From ddc6df79e8cc756a4d968b68bf488eec936b446a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:51:26 +0000 Subject: [PATCH 001/120] commitment: extract canonical DecodeBranchInto from unfoldBranchNode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure refactor — behavior preserved. Splits the encoded-branch→cells decode logic out of HexPatriciaHashed.unfoldBranchNode into a free function DecodeBranchInto so the same code is consumed by: - unfoldBranchNode (existing trie unfold path) - future cache populators (decoded-payload BranchCache) - future parallel pre-unfold orchestrator (Stage E) Today these would each have to re-derive the encoded-branch parsing logic. Centralising it ensures one decoder, one set of edge cases, one place to fix any bug in the on-disk format handling. DecodeBranchInto is intentionally PURE — it does not call deriveHashedKeys. Trie callers (which need hashed keys for the fold state machine) follow the decode with their own keccak loop. Cache callers can skip the derive step entirely until the cell is consumed by the trie. Tests: - TestDecodeBranchInto_RoundTrip: BranchEncoder.EncodeBranch produces bytes that DecodeBranchInto recovers cell-for-cell. Property test that keeps the canonical decoder consistent with the canonical encoder. - TestDecodeBranchInto_DeletedFlag: touchMap/afterMap convention with the deleted parameter. - TestDecodeBranchInto_TruncatedInput: clean errors on truncated input (no panic). Plus the existing commitment test suite (incl. trie-mismatch tests in TestBranchData_*) all pass without modification, confirming the refactor preserves unfoldBranchNode's behavior. This is the foundation for the next refactors in the representation-reduction track (see agentspecs/trie-data-pipeline-complexity-tax.md): subsequent PRs will introduce a decoded-payload cache that reads through this same decoder, and will lift unfoldKeyPath as a per-key traversal primitive that the warmer + future Stage E both consume. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_decode.go | 95 ++++++++++++++++++ execution/commitment/branch_decode_test.go | 106 ++++++++++++++++++++ execution/commitment/hex_patricia_hashed.go | 21 ++-- 3 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 execution/commitment/branch_decode.go create mode 100644 execution/commitment/branch_decode_test.go diff --git a/execution/commitment/branch_decode.go b/execution/commitment/branch_decode.go new file mode 100644 index 00000000000..0e4c6b5ccc9 --- /dev/null +++ b/execution/commitment/branch_decode.go @@ -0,0 +1,95 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "encoding/binary" + "fmt" + "math/bits" +) + +// BranchMaps captures the three branch-level bitmasks decoded alongside the +// per-cell payload. Held separately from the cells slice so callers can apply +// them into trie state without struct-assigning the whole [16]cell array. +type BranchMaps struct { + Bitmap uint16 // present-children bitmap (the canonical map encoded in the branch) + TouchMap uint16 // children that were touched in this branch (set by caller per deleted flag) + AfterMap uint16 // children present after this commitment step +} + +// DecodeBranchInto parses the on-disk encoded form of a branch (as returned +// by ctx.Branch / readBranchAndCheckForFlushing with the leading 2-byte +// touch-map stripped) and populates the supplied cells array. +// +// branchData must already have the leading touch-map prefix stripped — the +// raw bytes returned from Branch() include it; callers strip it before +// invoking this function (matching the unfoldBranchNode call pattern). +// +// deleted controls the touchMap/afterMap convention: +// - deleted=true → all decoded cells are touched-but-not-present-after +// (touchMap = bitmap, afterMap = 0) +// - deleted=false → all decoded cells are present-after (touchMap = 0, +// afterMap = bitmap) +// +// Returns the three branch-level maps for the caller to apply into their +// own state machine. The cells array is mutated in place. +// +// This is a PURE decode — it does NOT call deriveHashedKeys on the cells. +// Trie callers (HexPatriciaHashed.unfoldBranchNode) follow this with their +// own loop calling deriveHashedKeys per cell, since they have the keccak +// scratch buffer. Cache callers can skip deriveHashedKeys entirely until +// the cell is consumed by the trie. +// +// This function is the canonical decoder for branch data. Both +// unfoldBranchNode (which feeds the in-memory grid for fold/unfold) and +// future cache populators consume branches via this same path so the +// encoded→cells transformation lives in one place. +func DecodeBranchInto( + branchData []byte, + deleted bool, + cells *[16]cell, +) (BranchMaps, error) { + if len(branchData) < 2 { + return BranchMaps{}, fmt.Errorf("branch data too short for bitmap: %d bytes", len(branchData)) + } + bitmap := binary.BigEndian.Uint16(branchData[0:]) + maps := BranchMaps{Bitmap: bitmap} + if deleted { + maps.TouchMap, maps.AfterMap = bitmap, 0 + } else { + maps.TouchMap, maps.AfterMap = 0, bitmap + } + + pos := 2 + for bitset := bitmap; bitset != 0; { + bit := bitset & -bitset + nibble := bits.TrailingZeros16(bit) + c := &cells[nibble] + if pos >= len(branchData) { + return BranchMaps{}, fmt.Errorf("branch data truncated before cell at nibble %d", nibble) + } + fieldBits := branchData[pos] + pos++ + newPos, err := c.fillFromFields(branchData, pos, cellFields(fieldBits)) + if err != nil { + return BranchMaps{}, fmt.Errorf("fillFromFields nibble %d: %w", nibble, err) + } + pos = newPos + bitset ^= bit + } + return maps, nil +} diff --git a/execution/commitment/branch_decode_test.go b/execution/commitment/branch_decode_test.go new file mode 100644 index 00000000000..abe80e0b25f --- /dev/null +++ b/execution/commitment/branch_decode_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDecodeBranchInto_RoundTrip asserts that DecodeBranchInto recovers +// the cells encoded by BranchEncoder.EncodeBranch — the property test that +// keeps the canonical decoder consistent with the canonical encoder. +func TestDecodeBranchInto_RoundTrip(t *testing.T) { + t.Parallel() + row, bm := generateCellRow(t, 16) + + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + require.NotEmpty(t, enc) + + // EncodeBranch produces bytes WITH the 2-byte touch-map prefix; the + // decoder consumes the bytes WITHOUT it (matching the unfoldBranchNode + // call pattern, which strips the touch-map prefix before decoding). + branchData := []byte(enc)[2:] + + var cells [16]cell + maps, err := DecodeBranchInto(branchData, false /* not deleted */, &cells) + require.NoError(t, err) + + // Bitmap should match what was encoded + require.Equal(t, bm, maps.Bitmap, "decoded bitmap mismatch") + require.Equal(t, uint16(0), maps.TouchMap, "expected empty touchMap when deleted=false") + require.Equal(t, bm, maps.AfterMap, "afterMap should equal bitmap when deleted=false") + + // Each present cell should match the original on the fields that + // survive encode→decode (extension, account/storage addr, hash). + // hashedExtension etc. are set by deriveHashedKeys (separate step) and + // are not part of the decoder's responsibility. + for i, orig := range row { + decoded := &cells[i] + require.Equal(t, orig.extLen, decoded.extLen, "cell %d extLen", i) + require.Equal(t, orig.extension[:orig.extLen], decoded.extension[:decoded.extLen], "cell %d extension", i) + require.Equal(t, orig.accountAddrLen, decoded.accountAddrLen, "cell %d accountAddrLen", i) + require.Equal(t, orig.accountAddr[:orig.accountAddrLen], decoded.accountAddr[:decoded.accountAddrLen], "cell %d accountAddr", i) + require.Equal(t, orig.storageAddrLen, decoded.storageAddrLen, "cell %d storageAddrLen", i) + require.Equal(t, orig.storageAddr[:orig.storageAddrLen], decoded.storageAddr[:decoded.storageAddrLen], "cell %d storageAddr", i) + require.Equal(t, orig.hashLen, decoded.hashLen, "cell %d hashLen", i) + require.Equal(t, orig.hash[:orig.hashLen], decoded.hash[:decoded.hashLen], "cell %d hash", i) + } +} + +// TestDecodeBranchInto_DeletedFlag verifies the touchMap/afterMap convention +// flips correctly with the deleted parameter. +func TestDecodeBranchInto_DeletedFlag(t *testing.T) { + t.Parallel() + row, bm := generateCellRow(t, 16) + + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + branchData := []byte(enc)[2:] + + var cells [16]cell + maps, err := DecodeBranchInto(branchData, true, &cells) + require.NoError(t, err) + require.Equal(t, bm, maps.Bitmap) + require.Equal(t, bm, maps.TouchMap, "deleted=true → touchMap = bitmap") + require.Equal(t, uint16(0), maps.AfterMap, "deleted=true → afterMap = 0") +} + +// TestDecodeBranchInto_TruncatedInput asserts the decoder fails cleanly on +// truncated branch data instead of panicking. +func TestDecodeBranchInto_TruncatedInput(t *testing.T) { + t.Parallel() + var cells [16]cell + + // Empty data — should fail at bitmap read. + _, err := DecodeBranchInto(nil, false, &cells) + require.Error(t, err) + + // Just bitmap, no cells — should be fine if bitmap is 0. + _, err = DecodeBranchInto([]byte{0x00, 0x00}, false, &cells) + require.NoError(t, err, "bitmap=0 with no cell data should decode cleanly") + + // Bitmap claims one cell but data missing — should fail. + _, err = DecodeBranchInto([]byte{0x00, 0x01}, false, &cells) + require.Error(t, err, "bitmap with set bit but no cell data should error") +} diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 887e9fb8233..09f4540c495 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1750,28 +1750,19 @@ func (hph *HexPatriciaHashed) unfoldBranchNode(row int, depth int16, deleted boo return fmt.Errorf("empty branch data read during unfold, compact prefix %x nibbles %x", key, hph.currentKey[:hph.currentKeyLen]) } hph.branchBefore[row] = true - bitmap := binary.BigEndian.Uint16(branchData[0:]) - pos := 2 - if deleted { // All cells come as deleted (touched but not present after) - hph.touchMap[row], hph.afterMap[row] = bitmap, 0 - } else { - hph.touchMap[row], hph.afterMap[row] = 0, bitmap + maps, err := DecodeBranchInto(branchData, deleted, &hph.grid[row]) + if err != nil { + return fmt.Errorf("prefix [%x] branchData[%x]: %w", hph.currentKey[:hph.currentKeyLen], branchData, err) } - //fmt.Printf("unfoldBranchNode prefix '%x' [%x], afterMap = [%016b], touchMap = [%016b]\n", key, branchData, hph.afterMap[row], hph.touchMap[row]) - // Loop iterating over the set bits of modMask - for bitset, j := bitmap, 0; bitset != 0; j++ { + hph.touchMap[row] = maps.TouchMap + hph.afterMap[row] = maps.AfterMap + for bitset := maps.Bitmap; bitset != 0; { bit := bitset & -bitset nibble := bits.TrailingZeros16(bit) cell := &hph.grid[row][nibble] - fieldBits := branchData[pos] - pos++ - if pos, err = cell.fillFromFields(branchData, pos, cellFields(fieldBits)); err != nil { - return fmt.Errorf("prefix [%x] branchData[%x]: %w", hph.currentKey[:hph.currentKeyLen], branchData, err) - } if hph.trace { fmt.Printf("cell (%d, %x, depth=%d) %s\n", row, nibble, depth, cell.FullString()) } - // relies on plain account/storage key so need to be dereferenced before hashing if err = cell.deriveHashedKeys(depth, hph.keccak, hph.accountKeyLen, hph.cellHashBuf[:]); err != nil { return err From 31fdf5789338ef208eb48a66d105b62e421b6dac Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:59:21 +0000 Subject: [PATCH 002/120] commitment: add BranchCache with pinned root + bounded LRU tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a new BranchCache type, distinct from WarmupCache and designed for the longer-lived caching the cross-block persistence work needs (step 7 of the representation-reduction sequence). Distinguishing characteristics vs WarmupCache: - Bounded LRU tail with configurable capacity (vs WarmupCache's unbounded map). Suitable for caches that outlive a single Process without unbounded memory growth. - Single pinned slot for the root branch (compact prefix [0x00]). Root never evicts. Atomic-pointer load/store on the hot read path, no lock involved. - dirty-flag + PutIfClean invariants — same semantics as the invariants added to WarmupCache in the previous commit. Lets cross-block writers race safely with fold updates. - Lazy GetDecoded — same lazy-decode pattern as WarmupCache's GetBranchDecoded; cells populated on first decoded-read and cached for subsequent reads. NOT yet wired into the trie's read or write paths. This commit just adds the type, with tests. The trie integration (where this cache plugs into branchFromCacheOrDB and the encoder's PutBranch) is the discussion point at the step 6 boundary — see the conversation captured at this point in the representation-reduction sequence. Today the cache is intended to be ephemeral (per-Process, constructed alongside the trie, dies with it). Step 7 lifts the lifetime to the aggTx level for cross-block persistence; the cache shape (bounded LRU + pinned root + dirty-flag) is in place ahead of that. Tests: - TestBranchCache_RootPinning: root branch lands in pinned slot, deep branches land in LRU tail; per-tier hit counters update independently. - TestBranchCache_RootSurvivesEvictionPressure: root persists when tail is overfilled past capacity. - TestBranchCache_DirtyFlag: PutIfClean refuses dirty entry, unconditional Put replaces and clears dirty. - TestBranchCache_GetDecoded: lazy-decode round-trip with BranchEncoder; cells pointer reused across reads. - TestBranchCache_Invalidate: removes from both tiers. - TestBranchCache_Clear: empties both tiers, resets stats. - TestBranchCache_Stats: deterministic format with per-tier counts. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 272 ++++++++++++++++++++++ execution/commitment/branch_cache_test.go | 185 +++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 execution/commitment/branch_cache.go create mode 100644 execution/commitment/branch_cache_test.go diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go new file mode 100644 index 00000000000..9bea8200207 --- /dev/null +++ b/execution/commitment/branch_cache.go @@ -0,0 +1,272 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/erigontech/erigon/common/maphash" +) + +// BranchCache stores commitment-trie branch data with a persistence-friendly +// shape distinct from WarmupCache: +// +// - Bounded LRU tail with configurable capacity (eviction is well-defined, +// suitable for long-lived caching across many Process calls without +// unbounded memory growth). +// - Single pinned slot for the root branch (always hottest, always present +// once populated, never subject to LRU eviction). Compact prefix of +// length 0 (or single-byte "no-key" form) targets this slot. +// - dirty-flag + PutIfClean invariants (carried from step 4 of the +// representation-reduction sequence) so cross-block writers can race +// safely with fold updates. +// - Lazy-decoded read path (GetDecoded) — same pattern as WarmupCache's +// GetBranchDecoded; cells are populated from the cached encoded form +// on first decoded-read and reused thereafter. +// +// Today this is constructed as ephemeral infrastructure (per-Process, +// alongside the trie). Future cross-block persistence work (step 7 of +// the representation-reduction sequence) will lift its lifetime to the +// aggTx level. The wire-up between the trie's read path and BranchCache +// is intentionally NOT done in this commit — see the integration +// discussion captured at the time of step 6. +// +// Distinct from WarmupCache: WarmupCache is a simple unbounded map for +// warmup-fill hot-read patterns within one Process. BranchCache is +// designed for the longer-lived cache lifetime that the cross-block +// persistence work needs. +type BranchCache struct { + // Pinned tier — single slot for the root branch. Atomic-pointer + // access so no lock is needed for the hot read path. + root atomic.Pointer[branchCacheEntry] + + // LRU tail — bounded entries, evicts oldest when full. maphash.LRU + // wraps hashicorp/golang-lru/v2 which is thread-safe internally. + tail *maphash.LRU[*branchCacheEntry] + + // Stats — atomic counters surfaced via Stats(). + rootHits, rootMisses atomic.Uint64 + tailHits, tailMisses atomic.Uint64 + bytesServed atomic.Uint64 +} + +type branchCacheEntry struct { + // data is the canonical encoded form (with the leading 2-byte touch-map + // prefix). Always populated by Put / PutIfClean. + data []byte + + // Lazy-decoded form. Populated on first GetDecoded for this entry; + // subsequent reads return the cached cells. decodeOnce ensures decode + // runs at most once even under concurrent reads. + decodeOnce sync.Once + cells [16]cell + cellsBitmap uint16 + decodedReady bool + decodeErr error + + // dirty signals "the canonical store has been written to since this + // entry was populated; treat as stale until cleared." Same semantics + // as branchEntry.dirty in WarmupCache (carried from step 4). + dirty atomic.Bool +} + +// DefaultBranchCacheTailCapacity is the LRU tail size used when no +// explicit capacity is given. ~50k entries × ~500 bytes = ~25 MB +// at typical mainnet branch sizes. +const DefaultBranchCacheTailCapacity = 50000 + +// NewBranchCache constructs a BranchCache with the given LRU tail capacity. +// Capacity <= 0 panics — pass a positive value or DefaultBranchCacheTailCapacity. +func NewBranchCache(tailCapacity int) *BranchCache { + if tailCapacity <= 0 { + panic(fmt.Sprintf("BranchCache: tailCapacity must be positive, got %d", tailCapacity)) + } + tail, err := maphash.NewLRU[*branchCacheEntry](tailCapacity) + if err != nil { + panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) + } + return &BranchCache{tail: tail} +} + +// isRootPrefix reports whether prefix targets the pinned root slot. The +// commitment-trie compact encoding uses a 1-byte even-length flag (0x00) +// to represent the empty nibble path (root branch). Anything longer goes +// to the LRU tail. +func isRootPrefix(prefix []byte) bool { + return len(prefix) == 1 && prefix[0] == 0x00 +} + +func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { + if isRootPrefix(prefix) { + entry := c.root.Load() + if entry == nil { + c.rootMisses.Add(1) + return nil, false + } + c.rootHits.Add(1) + return entry, true + } + entry, ok := c.tail.Get(prefix) + if !ok { + c.tailMisses.Add(1) + return nil, false + } + c.tailHits.Add(1) + return entry, true +} + +func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { + if isRootPrefix(prefix) { + c.root.Store(entry) + return + } + c.tail.Set(prefix, entry) +} + +// Get retrieves branch data from the cache. Returns the canonical encoded +// bytes (with the leading 2-byte touch-map prefix). +func (c *BranchCache) Get(prefix []byte) ([]byte, bool) { + entry, ok := c.lookup(prefix) + if !ok { + return nil, false + } + c.bytesServed.Add(uint64(len(entry.data))) + return entry.data, true +} + +// GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on +// first access for each entry; subsequent reads return the cached cells +// pointer without redoing the parse work. +// +// Returns the bitmap of present children plus a pointer to the populated +// cells array. Caller derives touchMap/afterMap based on its own context +// (deleted vs present-after) — same convention as WarmupCache.GetBranchDecoded. +// +// The returned *[16]cell aliases storage owned by the cache entry — the +// caller MUST NOT modify the cells in place. Read-only consumption is +// safe across concurrent calls. +func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + entry, found := c.lookup(prefix) + if !found { + return 0, nil, false + } + entry.decodeOnce.Do(func() { + if len(entry.data) < 2 { + entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) + return + } + maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) + if err != nil { + entry.decodeErr = err + return + } + entry.cellsBitmap = maps.Bitmap + entry.decodedReady = true + }) + if !entry.decodedReady { + return 0, nil, false + } + c.bytesServed.Add(uint64(len(entry.data))) + return entry.cellsBitmap, &entry.cells, true +} + +// Put stores branch data in the cache, replacing any existing entry +// (clearing its dirty flag in the process — the new entry is fresh). +// Always copies the input data so the cache owns it independently of +// caller buffer lifetime. +func (c *BranchCache) Put(prefix []byte, data []byte) { + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.store(prefix, &branchCacheEntry{data: dataCopy}) +} + +// PutIfClean stores branch data only if no existing entry is marked dirty. +// Returns true on store, false if a dirty entry was present (indicating a +// canonical-store write is in progress and the caller's data is potentially +// stale). +// +// Same semantics as WarmupCache.PutBranchIfClean — see that doc for the +// race-it-protects-against narrative. +func (c *BranchCache) PutIfClean(prefix []byte, data []byte) bool { + if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { + return false + } + c.Put(prefix, data) + return true +} + +// MarkDirty flags the entry at prefix as stale-until-cleared. Subsequent +// PutIfClean calls for this prefix will skip; reads still return the +// entry (the dirty signal is consumed only on the write path today, same +// as WarmupCache.MarkBranchDirty). +// +// No-op if no entry exists at prefix. +func (c *BranchCache) MarkDirty(prefix []byte) { + if entry, ok := c.lookup(prefix); ok { + entry.dirty.Store(true) + } +} + +// Invalidate removes the entry at prefix entirely. Use when the caller +// knows the canonical store has changed and the cached entry should not +// be served at all (vs MarkDirty which keeps the entry but blocks +// PutIfClean overwrites). +func (c *BranchCache) Invalidate(prefix []byte) { + if isRootPrefix(prefix) { + c.root.Store(nil) + return + } + c.tail.Delete(prefix) +} + +// Clear empties the cache and resets stats counters. Use on Reset / +// fork-validation paths to ensure stale entries from one trie root are +// not served against a different root. +func (c *BranchCache) Clear() { + c.root.Store(nil) + c.tail.Purge() + c.rootHits.Store(0) + c.rootMisses.Store(0) + c.tailHits.Store(0) + c.tailMisses.Store(0) + c.bytesServed.Store(0) +} + +// Stats returns a one-line summary of root-tier and tail-tier hit/miss +// counters plus bytes served. Format mirrors WarmupCache.Stats() so +// per-Process log lines can compose them. +func (c *BranchCache) Stats() string { + rh, rm := c.rootHits.Load(), c.rootMisses.Load() + th, tm := c.tailHits.Load(), c.tailMisses.Load() + bb := c.bytesServed.Load() + pct := func(hit, miss uint64) float64 { + total := hit + miss + if total == 0 { + return 0 + } + return 100.0 * float64(hit) / float64(total) + } + return fmt.Sprintf( + "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d", + rh, rm, pct(rh, rm), + th, tm, pct(th, tm), + float64(bb)/1024/1024, + c.tail.Len(), + ) +} diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go new file mode 100644 index 00000000000..5338a090701 --- /dev/null +++ b/execution/commitment/branch_cache_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBranchCache_RootPinning verifies the root branch lands in the pinned +// slot (counted as root-hit) and tail entries land in the LRU tier +// (counted as tail-hit). +func TestBranchCache_RootPinning(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch + deepKey := []byte{0x12, 0x34, 0x56} + c.Put(rootKey, []byte("root-data")) + c.Put(deepKey, []byte("deep-data")) + + // Root reads should increment rootHits, not tailHits + got, ok := c.Get(rootKey) + require.True(t, ok) + require.Equal(t, []byte("root-data"), got) + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(0), c.tailHits.Load()) + + // Deep reads should increment tailHits, not rootHits + got, ok = c.Get(deepKey) + require.True(t, ok) + require.Equal(t, []byte("deep-data"), got) + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(1), c.tailHits.Load()) +} + +// TestBranchCache_RootSurvivesEvictionPressure verifies that pinned root +// entry is not subject to LRU eviction even if the tail fills past +// capacity many times over. +func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { + c := NewBranchCache(10) // very small tail + rootKey := []byte{0x00} + c.Put(rootKey, []byte("ROOT-PERSISTS")) + + // Stuff the tail well past capacity + for i := 0; i < 100; i++ { + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}) + } + + // Root must still be there + got, ok := c.Get(rootKey) + require.True(t, ok, "root should never be evicted from pinned slot") + require.Equal(t, []byte("ROOT-PERSISTS"), got) + + // Tail at capacity (10), not 100 + require.LessOrEqual(t, c.tail.Len(), 10, "tail should respect LRU capacity") +} + +// TestBranchCache_DirtyFlag verifies PutIfClean refuses overwrite of a +// dirty entry, while Put unconditionally replaces (and the new entry +// starts clean). +func TestBranchCache_DirtyFlag(t *testing.T) { + c := NewBranchCache(100) + key := []byte{0x12, 0x34} + + require.True(t, c.PutIfClean(key, []byte("v1"))) + c.MarkDirty(key) + + require.False(t, c.PutIfClean(key, []byte("v2")), "PutIfClean must refuse dirty entry") + got, _ := c.Get(key) + require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") + + // Unconditional Put replaces + c.Put(key, []byte("v3")) + got, _ = c.Get(key) + require.Equal(t, []byte("v3"), got) + + // New entry is clean + require.True(t, c.PutIfClean(key, []byte("v4"))) +} + +// TestBranchCache_GetDecoded verifies the lazy-decode read path for a +// real encoded branch (round-trip with BranchEncoder). +func TestBranchCache_GetDecoded(t *testing.T) { + c := NewBranchCache(100) + + row, bm := generateCellRow(t, 16) + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + + prefix := []byte{0x12, 0x34} + c.Put(prefix, enc) + + // First decoded-read decodes lazily + bitmap, cells, ok := c.GetDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap) + require.NotNil(t, cells) + + // Second call returns same cells pointer + _, cells2, ok := c.GetDecoded(prefix) + require.True(t, ok) + require.Same(t, cells, cells2, "decoded form cached and reused") + + // Encoded form unchanged after decoded reads + encGot, ok := c.Get(prefix) + require.True(t, ok) + require.Equal(t, []byte(enc), encGot) +} + +// TestBranchCache_Invalidate removes entries from both tiers. +func TestBranchCache_Invalidate(t *testing.T) { + c := NewBranchCache(100) + rootKey := []byte{0x00} + deepKey := []byte{0x12, 0x34} + c.Put(rootKey, []byte("r")) + c.Put(deepKey, []byte("d")) + + c.Invalidate(rootKey) + _, ok := c.Get(rootKey) + require.False(t, ok, "root invalidated") + + c.Invalidate(deepKey) + _, ok = c.Get(deepKey) + require.False(t, ok, "deep invalidated") +} + +// TestBranchCache_Clear empties everything and resets stats. +func TestBranchCache_Clear(t *testing.T) { + c := NewBranchCache(100) + c.Put([]byte{0x00}, []byte("r")) + c.Put([]byte{0x12}, []byte("d")) + c.Get([]byte{0x00}) + c.Get([]byte{0x12}) + + require.Equal(t, uint64(1), c.rootHits.Load()) + require.Equal(t, uint64(1), c.tailHits.Load()) + + c.Clear() + require.Equal(t, uint64(0), c.rootHits.Load()) + require.Equal(t, uint64(0), c.tailHits.Load()) + _, ok := c.Get([]byte{0x00}) + require.False(t, ok) + _, ok = c.Get([]byte{0x12}) + require.False(t, ok) +} + +// TestBranchCache_Stats verifies the format of the stats string is +// deterministic and contains the expected per-tier counts. +func TestBranchCache_Stats(t *testing.T) { + c := NewBranchCache(100) + c.Put([]byte{0x00}, []byte("rrr")) + c.Put([]byte{0x12, 0x34}, []byte("ddd")) + c.Get([]byte{0x00}) + c.Get([]byte{0x12, 0x34}) + c.Get([]byte{0xff}) // tail miss + + s := c.Stats() + for _, want := range []string{ + "root hit=1 miss=0", + "tail hit=1 miss=1", + "tail entries=1", // we put 1 deep entry; root not counted in tail + } { + require.Contains(t, s, want, "Stats output: %s", s) + } + // Sanity: format doesn't blow up if we read it + require.True(t, strings.HasPrefix(s, "branch-cache ")) +} From 1593f0e8c0eb18c4f6aa839aff7ecf83ed4ce4ce Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 09:16:24 +0000 Subject: [PATCH 003/120] commitment: document BranchCache concurrency contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc-only addition to BranchCache's package-level comment, capturing the caller invariants the cache assumes and the conditions under which the existing concurrent trie satisfies them. Three caller invariants: 1. Single writer per prefix at any moment. 2. Mark-dirty-then-Put discipline for racing writers. 3. Decoded cells from GetDecoded are read-only (alias entry storage). The current ConcurrentPatriciaHashed satisfies all three by construction: - Mounts partition by first nibble (disjoint prefix spaces, no cross-mount writes to the same key). - Root branch written by single sequential fold post-Wait. - Mount→root grid roll-up is rootMu-protected (in-memory grid only, separate from cache writes). Doc explicitly flags that any future parallel fold redesign (Stage F in agentspecs/stage-e-pre-unfold-design.md) MUST preserve these invariants — particularly the single-writer-per-prefix one, which breaks if parent branches are written incrementally as children complete in parallel. The required coordination layer goes at the orchestrator (per-parent atomic counter; only the last-decrementer writes the parent), NOT inside the cache. The cache's existing primitives (atomic dirty flag, thread-safe LRU, atomic root pointer) are sufficient for that orchestrator to build on. Motivation: Stage F is likely deferred because the bench data shows fold isn't the bottleneck for the canonical SSTORE-bloat workload. But adding the constraint to the cache later (after caching is in production) is much harder than documenting it now — correctness regressions from a missed coordination layer can hide for many blocks. Documenting the contract on the cache itself ensures any engineer touching parallel fold sees it. No code change. Doc-only. All tests still pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9bea8200207..4851fc7bb5e 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -51,6 +51,79 @@ import ( // warmup-fill hot-read patterns within one Process. BranchCache is // designed for the longer-lived cache lifetime that the cross-block // persistence work needs. +// +// # Concurrency contract — caller invariants +// +// Internally, the LRU tail is thread-safe (hashicorp/golang-lru/v2) and +// the pinned root slot is an atomic.Pointer. So any combination of +// concurrent Get / GetDecoded / Put / PutIfClean / MarkDirty / Invalidate +// is mechanically safe — no panics, no torn reads. But "mechanically safe" +// is NOT the same as "logically consistent across writers." The cache is +// designed to be used under the following caller invariants: +// +// 1. Single writer per prefix at any moment. The cache does not coordinate +// concurrent writes to the same key — last-Put-wins semantics, with no +// guarantee that the winning value is the one the application wanted. +// +// 2. Mark-dirty-then-Put discipline for writers that may race with +// readers. Caller calls MarkDirty BEFORE producing the new bytes, then +// Put AFTER the canonical-store write succeeds. This is the +// deferred-encoding-friendly alternative to inline invalidation +// (motivated by the prototype investigation that found inline +// invalidate is incompatible with deferred encoding — see +// agentspecs/commitment-cache-prototype-dev-context.md). +// +// 3. Decoded cells returned by GetDecoded MUST NOT be mutated. The +// *[16]cell pointer aliases entry-owned storage and is shared across +// readers; in-place mutation breaks consistency for all subsequent +// readers of that prefix. +// +// # Concurrency contract — how the existing concurrent trie satisfies it +// +// The current ConcurrentPatriciaHashed (parallel commitment calculator) +// satisfies all three caller invariants by construction: +// +// - Mounts partition the prefix space by FIRST NIBBLE. Mount N's +// encoder only writes branches whose key starts with [0x0N ...]. +// Different mounts therefore never write to the same prefix. +// (See hex_concurrent_patricia_hashed.go: NewConcurrentPatriciaHashed +// creates 16 mounts via SpawnSubTrie; each mount has its own HPH, +// own BranchEncoder, own PatriciaContext / roTx.) +// +// - Root branch (prefix [0x00]) is written by the single root fold +// that runs SEQUENTIALLY after errgroup.Wait() in ParallelHashSort. +// One writer for the pinned root slot. +// +// - Mount→root grid roll-up is mutex-protected via +// ConcurrentPatriciaHashed.rootMu — but that updates IN-MEMORY grid +// cells, not the cache. The cache only sees the eventual root +// branch when the post-Wait root fold encodes it. +// +// # Concurrency contract — what future parallel fold work must preserve +// +// Stage F (parallel tree-reduce fold), described in +// agentspecs/trie-data-pipeline-complexity-tax.md, would change condition +// 2 above: the parent fold (incl. root) would no longer be a single +// post-Wait sequential pass. Multiple goroutines would compute parent +// branches in parallel as their children complete. This MUST not violate +// "single writer per prefix" — any future Stage F design needs an +// explicit per-prefix coordination layer (atomic counter on parent +// "children remaining"; only the last-decrementer writes the parent). +// The dirty-flag + PutIfClean primitives in this cache are sufficient +// for that coordination layer; the cache itself does NOT add per-prefix +// locking because that would be wasted work for the current architecture. +// +// If you are implementing parallel fold (or any other architecture that +// breaks the "single writer per prefix" invariant), do NOT relax the +// invariant by adding internal locking to the cache. Add the +// coordination at the orchestrator layer where the partitioning logic +// lives. The cache stays simple; the orchestrator owns the discipline. +// +// Likewise if you change the prefix partitioning (e.g. by-second-nibble +// mounts, depth-based partitioning, anything other than first-nibble), +// re-validate that distinct workers continue to write disjoint prefix +// spaces. Re-read the partitioning code in +// hex_concurrent_patricia_hashed.go and confirm. type BranchCache struct { // Pinned tier — single slot for the root branch. Atomic-pointer // access so no lock is needed for the hot read path. From 8a2c56513e2bc859d8bec9527bd77cf12d77f43d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 09:28:34 +0000 Subject: [PATCH 004/120] commitment: wire BranchCache into trie read + encoder write paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 7a of the representation-reduction sequence: per-Process integration of the BranchCache type added in the previous commit. Plumbing: - Trie interface gains SetBranchCache(*BranchCache). HexPatriciaHashed implementation propagates to its branchEncoder. ConcurrentPatriciaHashed implementation propagates the SAME instance to root + all 16 mounts (sharing one cache is correct under the concurrency contract — mounts partition prefix space by first nibble, so cross-mount writes target distinct keys). - InitializeTrieAndUpdates constructs a new BranchCache(default) per trie instance and attaches it. Lifetime today = trie lifetime = per-Process. Future cross-block persistence work (step 7b) lifts this to aggTx scope by constructing the cache one layer up and passing it in. Read path (HPH.branchFromCacheOrDB): L1 WarmupCache (existing) → L2 BranchCache (new) → L3 ctx.Branch. L3 hits with non-empty result populate L2 so subsequent reads hit L2 within the cache's lifetime. L1 stays first because warmup workers may have pre-fetched with prefix-walk-derived freshness. Write path (BranchEncoder.CollectUpdate): - MarkDirty(prefix) BEFORE encode work — protects against concurrent warmup-style writers racing into PutIfClean during the encode (race documented in the cache's Concurrency Contract). - Put(prefixCopy, updateCopy) AFTER ctx.PutBranch succeeds — replaces the dirty entry with fresh canonical bytes. Single writer per prefix per fold step (current sequential fold + first-nibble mount partitioning) means no race on this Put. Lifecycle: HPH.Reset clears the BranchCache when called from the root trie (gated by !hph.mounted). Mounted subtries share the root's cache, so a mount calling Clear would dump entries the root still wants. Carries the invariant from PR #19954 commit 1612d5608c. Today's expected performance impact: minimal. Per-Process lifetime means cache is empty at Process start, so first reads always miss. The cache helps only branches that are read multiple times within ONE Process — uncommon in current code paths. Step 7b is where the real perf swing comes from (cross-block persistence so block N reads hit branches written by block N-1). This commit is the safe stepping stone: it validates the wire-up end-to-end (read path + write path + concurrency contract + lifecycle) without changing perf characteristics. Bench should match Run I baseline (7.16 mgas/s on canonical SSTORE-bloat block). All commitment tests pass, lint clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/commitment.go | 28 +++++++++ .../hex_concurrent_patricia_hashed.go | 12 ++++ execution/commitment/hex_patricia_hashed.go | 57 ++++++++++++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ee730aee321..727fafef409 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -102,6 +102,12 @@ type Trie interface { EnableCsvMetrics(filePathPrefix string) // EnableWarmupCache enables/disables warmup cache during Process (false by default) EnableWarmupCache(bool) + // SetBranchCache attaches a BranchCache instance for branch read/write + // caching across the trie + branchEncoder. ConcurrentPatriciaHashed + // implementations propagate the same instance to all mounts so the + // concurrency contract documented in branch_cache.go is respected + // (single shared cache, partitioned-by-prefix writers). + SetBranchCache(*BranchCache) // Variant returns commitment trie variant Variant() TrieVariant @@ -155,6 +161,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * case VariantConcurrentHexPatricia: root := NewHexPatriciaHashed(length.Addr, nil) trie := NewConcurrentPatriciaHashed(root, nil) + trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) // tree.SetConcurrentCommitment(true) // first run always sequential return trie, tree @@ -169,6 +176,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * default: trie := NewHexPatriciaHashed(length.Addr, nil) + trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) return trie, tree } @@ -350,6 +358,10 @@ type BranchEncoder struct { deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates cache *WarmupCache + // branchCache, when set, receives mark-dirty-before-encode + Put-after- + // canonical-write so the cache stays consistent with ctx.PutBranch. + // Set via HexPatriciaHashed.SetBranchCache (which propagates here). + branchCache *BranchCache } func NewBranchEncoder(sz uint64) *BranchEncoder { @@ -570,6 +582,15 @@ func (be *BranchEncoder) CollectUpdate( var foundInCache bool var err error + // Mark the BranchCache entry dirty BEFORE doing the encode work. Any + // concurrent warmer-style writer that races into PutIfClean for this + // prefix between now and our final Put below will see the dirty flag + // and skip — preventing a stale read from overwriting our fresh + // canonical write. See branch_cache.go's Concurrency Contract. + if be.branchCache != nil { + be.branchCache.MarkDirty(prefix) + } + if be.cache != nil { prev, foundInCache = be.cache.GetAndEvictBranch(prefix) if foundInCache && be.metrics != nil { @@ -606,6 +627,13 @@ func (be *BranchEncoder) CollectUpdate( if be.cache != nil { be.cache.PutBranch(prefixCopy, updateCopy) } + // Put on BranchCache replaces the (now dirty) prior entry with the + // fresh canonical bytes — clears the dirty flag in the process + // (the new entry is born clean). Single writer per prefix per fold + // invariant means no concurrent Put races on this key. + if be.branchCache != nil { + be.branchCache.Put(prefixCopy, updateCopy) + } if be.metrics != nil { be.metrics.updateBranch.Add(1) } diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index a5f32c42717..1fca7759959 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -171,6 +171,18 @@ func (p *ConcurrentPatriciaHashed) EnableWarmupCache(b bool) { p.mounts[i].EnableWarmupCache(b) } } + +// SetBranchCache attaches the same BranchCache instance to the root trie +// and all 16 mounts. Sharing one cache across mounts is correct under the +// concurrency contract (branch_cache.go): mounts partition the prefix +// space by first nibble, so cross-mount writes always target distinct +// cache keys. +func (p *ConcurrentPatriciaHashed) SetBranchCache(c *BranchCache) { + p.root.SetBranchCache(c) + for i := range p.mounts { + p.mounts[i].SetBranchCache(c) + } +} func (p *ConcurrentPatriciaHashed) GetCapture(truncate bool) []string { capture := p.root.GetCapture(truncate) if truncate { diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 09f4540c495..1ce1a1452ad 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -105,6 +105,13 @@ type HexPatriciaHashed struct { cache *WarmupCache enableWarmupCache bool // if true, enables warmup cache during Process (false by default) + // branchCache is the BranchCache instance attached via SetBranchCache. + // Wired into branchFromCacheOrDB as Level-2 (between WarmupCache and DB) + // and into branchEncoder for write-back. Today created per-Process by + // InitializeTrieAndUpdates; future cross-block persistence work lifts + // this to aggTx scope. See branch_cache.go's concurrency contract. + branchCache *BranchCache + // leaveDeferredForCaller when true, Process() leaves deferred updates on the branchEncoder // for the caller to handle via TakeDeferredUpdates(). When false (default), Process() // applies deferred updates inline. @@ -2923,6 +2930,20 @@ func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } func (hph *HexPatriciaHashed) EnableWarmupCache(enable bool) { hph.enableWarmupCache = enable } +// SetBranchCache attaches a BranchCache for branch read-through and +// write-through. Also propagates to the trie's BranchEncoder so encoder +// writes update the cache via mark-dirty-then-Put (see CollectUpdate). +// +// nil disables — both this trie's branchFromCacheOrDB Level-2 and the +// encoder's cache write-back become no-ops. +func (hph *HexPatriciaHashed) SetBranchCache(c *BranchCache) { + hph.branchCache = c + hph.branchEncoder.branchCache = c +} + +// BranchCache returns the attached BranchCache, or nil if none is set. +func (hph *HexPatriciaHashed) BranchCache() *BranchCache { return hph.branchCache } + func (hph *HexPatriciaHashed) GetCapture(truncate bool) []string { capture := hph.capture if truncate { @@ -2983,6 +3004,14 @@ func (hph *HexPatriciaHashed) Reset() { hph.rootTouched = false hph.rootChecked = false hph.rootPresent = true + + // Clear the BranchCache only when this is the root trie (not a + // mounted subtrie). Mounted subtries share the root's cache; clearing + // from a mount would dump entries the root still expects to see. + // Carries the invariant established in PR #19954 commit 1612d5608c. + if hph.branchCache != nil && !hph.mounted { + hph.branchCache.Clear() + } } func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { @@ -2994,7 +3023,20 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { return hph.cache } -// branchFromCacheOrDB reads branch data from cache if available, otherwise from DB. +// branchFromCacheOrDB reads branch data via the cache hierarchy: +// - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) +// - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped +// once the cross-block-persistence step lands) +// - L3: ctx.Branch (canonical store) +// +// On L2 hit no further work happens. On L3 read with a non-empty result the +// bytes are populated into BranchCache so subsequent reads (within the cache's +// lifetime) hit L2 instead. +// +// L1 (WarmupCache) is intentionally checked FIRST: warmup workers may have +// pre-fetched the branch with prefix-walk-derived freshness guarantees +// (see warmuper.go), and L1 entries are short-lived enough that staleness +// is bounded by Process duration. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { if hph.cache != nil { if data, found := hph.cache.GetBranch(key); found { @@ -3007,8 +3049,19 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } + if hph.branchCache != nil { + if data, found := hph.branchCache.Get(key); found { + return data, nil + } + } data, _, err := hph.ctx.Branch(key) - return data, err + if err != nil { + return nil, err + } + if hph.branchCache != nil && len(data) > 0 { + hph.branchCache.Put(key, data) + } + return data, nil } // accountFromCacheOrDB reads account data from cache if available, otherwise from DB. From 2849165e1e62913449fe42f1aeaa6ff757f63482 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 17:53:12 +0000 Subject: [PATCH 005/120] commitment, db/state: lift BranchCache lifetime to aggregator scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the BranchCache was constructed inside InitializeTrieAndUpdates, giving it per-SharedDomains lifetime. SharedDomains is reconstructed for every batch / many tx boundaries — verified with logs in the prototype (921 fresh BranchCache constructions per bench run) — so the cache started cold every batch and never delivered the cross-block hits the design is about. Per-Process scope kept cache-cleanup correctness simple but defeated the whole point of caching. Place the cache on the commitment Domain struct (one cache per aggregator, matching the pattern on add_execution_context_with_caches where each domain owns its valueCache). ConfigureDomains attaches the cache once after domains are initialised — idempotent, lifetime = aggregator lifetime. AggregatorRoTx exposes BranchCache() returning the commitment domain's cache, so SharedDomains' construction path can fetch it without forcing db/state/execctx to import db/state (db/state already imports execctx via squeeze.go, so the reverse import would create a cycle). The placeholder commitment.BranchCacheProvider interface lets the SD construction path use a duck-typed type assertion on tx.AggTx() any. Plumb the cache through NewSharedDomainsCommitmentContext into InitializeTrieAndUpdates as an explicit parameter; nil falls back to a fresh per-init cache so test helpers without an aggregator still get a valid cache. Reset behaviour: HexPatriciaHashed.Reset no longer calls Clear on the cache. Aggregator-scope persistence requires the cache to survive Reset between commitment calculations. Callers that genuinely need to invalidate the cache (unwind, fork validation) now call ClearBranchCache explicitly. The bench is forward-only so this is a safe change for measurement; an explicit unwind clear path can land in a follow-up commit when needed. --- db/state/aggregator.go | 20 +++++++++++++++++++ db/state/domain.go | 15 ++++++++++++++ db/state/execctx/domain_shared.go | 12 ++++++++++- execution/commitment/branch_cache.go | 13 ++++++++++++ execution/commitment/commitment.go | 15 +++++++++++--- .../commitmentdb/commitment_context.go | 9 +++++++-- execution/commitment/hex_patricia_hashed.go | 18 ++++++++++++----- 7 files changed, 91 insertions(+), 11 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 0ce0f36bcd2..93ff05ed6d0 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -359,6 +359,14 @@ func (a *Aggregator) ConfigureDomains() error { } a.configured = true + // Attach the long-lived commitment branch cache to the commitment + // domain. Lifetime = aggregator lifetime; shared by every + // SharedDomains derived from this aggregator. Idempotent across + // repeated ConfigureDomains calls (early return above). + if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { + cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) + } + if a.disableFsync { for _, d := range a.d { if d != nil { @@ -2414,6 +2422,18 @@ func (a *Aggregator) BeginFilesRo() *AggregatorRoTx { return ac } +// BranchCache returns the long-lived commitment branch cache attached to +// this aggregator's commitment domain. Implements +// commitment.BranchCacheProvider so the SharedDomains construction path +// can fetch the cache via a tx.AggTx() type assertion without forcing +// db/state/execctx to import db/state (which would create a cycle). +func (at *AggregatorRoTx) BranchCache() *commitment.BranchCache { + if at.d[kv.CommitmentDomain] == nil { + return nil + } + return at.d[kv.CommitmentDomain].d.branchCache +} + func (at *AggregatorRoTx) Dirs() datadir.Dirs { return at.a.dirs } func (at *AggregatorRoTx) standaloneIIs() []*InvertedIndexRoTx { return at.iis[:at.iisCount] } diff --git a/db/state/domain.go b/db/state/domain.go index a61a1ce444a..d7b52ff7fbe 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -51,6 +51,7 @@ import ( "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/commitment" ) var ( @@ -88,6 +89,13 @@ type Domain struct { checker *DependencyIntegrityChecker + // branchCache is the long-lived commitment-trie branch cache pinned to + // the aggregator. Set in ConfigureDomains for the commitment domain + // only; nil for every other domain. Lifetime = aggregator lifetime, + // independent of any single tx, batch, or SharedDomains. See + // commitment.BranchCacheProvider for the SDs' access path. + branchCache *commitment.BranchCache + // _testBuildAccessorHook - test-only: called with the recsplit before the build loop in buildHashMapAccessor _testBuildAccessorHook func(rs *recsplit.RecSplit) } @@ -135,6 +143,13 @@ func (d *Domain) SetChecker(checker *DependencyIntegrityChecker) { d.checker = checker } +// BranchCache returns the long-lived commitment-trie branch cache attached +// to this domain. Non-nil only on the commitment domain. Lifetime is the +// owning Aggregator's lifetime. +func (d *Domain) BranchCache() *commitment.BranchCache { + return d.branchCache +} + func (d *Domain) kvNewFilePath(fromStep, toStep kv.Step) string { return filepath.Join(d.dirs.SnapDomain, fmt.Sprintf("%s-%s.%d-%d.kv", d.FileVersion.DataKV.String(), d.FilenameBase, fromStep, toStep)) } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 16868883e4b..71d206f5de6 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -180,7 +180,17 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } sd.mem = tx.Debug().NewMemBatch(&sd.metrics) - sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp) + + // Fetch the aggregator-scope branch cache (lives on the commitment + // Domain, shared across all SharedDomains derived from this + // aggregator). The duck-typed BranchCacheProvider lookup avoids + // importing db/state directly — db/state already imports execctx, so + // the reverse import would create a cycle. + var branchCache *commitment.BranchCache + if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { + branchCache = p.BranchCache() + } + sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 4851fc7bb5e..6497cfadb7f 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -164,6 +164,19 @@ type branchCacheEntry struct { // at typical mainnet branch sizes. const DefaultBranchCacheTailCapacity = 50000 +// BranchCacheProvider exposes the long-lived BranchCache attached to the +// commitment domain. Implemented by *db/state.AggregatorRoTx (via duck +// typing) so callers in the SharedDomains construction path can fetch the +// cache without forcing db/state/execctx to import db/state — that import +// would create a cycle since db/state imports execctx (squeeze.go, +// trie_reader_integration_test.go, …). +// +// Returning nil is permitted; callers MUST treat nil as "no shared cache, +// behave as if disabled" rather than panic. +type BranchCacheProvider interface { + BranchCache() *BranchCache +} + // NewBranchCache constructs a BranchCache with the given LRU tail capacity. // Capacity <= 0 panics — pass a positive value or DefaultBranchCacheTailCapacity. func NewBranchCache(tailCapacity int) *BranchCache { diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 727fafef409..ddedeb95010 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -156,12 +156,21 @@ const ( VariantConcurrentHexPatricia TrieVariant = "hex-concurrent-patricia-hashed" ) -func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, *Updates) { +// InitializeTrieAndUpdates constructs the trie + updates buffer for a +// SharedDomainsCommitmentContext. The branchCache argument is the +// long-lived (aggregator-scope) cache shared across SDs — pass the cache +// returned by BranchCacheProvider on the AggregatorRoTx. When nil (test +// helpers, isolated benchmarks), a fresh per-init cache is created so the +// trie still has a valid cache to read/write through. +func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string, branchCache *BranchCache) (Trie, *Updates) { + if branchCache == nil { + branchCache = NewBranchCache(DefaultBranchCacheTailCapacity) + } switch tv { case VariantConcurrentHexPatricia: root := NewHexPatriciaHashed(length.Addr, nil) trie := NewConcurrentPatriciaHashed(root, nil) - trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) + trie.SetBranchCache(branchCache) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) // tree.SetConcurrentCommitment(true) // first run always sequential return trie, tree @@ -176,7 +185,7 @@ func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string) (Trie, * default: trie := NewHexPatriciaHashed(length.Addr, nil) - trie.SetBranchCache(NewBranchCache(DefaultBranchCacheTailCapacity)) + trie.SetBranchCache(branchCache) tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash) return trie, tree } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 36836bd009b..2c59c686052 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -189,12 +189,17 @@ func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix strin sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } -func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string) *SharedDomainsCommitmentContext { +// NewSharedDomainsCommitmentContext constructs a per-SharedDomains +// commitment context. branchCache is the aggregator-scope cache shared +// across all SDs derived from the same Aggregator; pass nil from test +// helpers that don't have an aggregator (a fresh per-init cache will be +// created inside InitializeTrieAndUpdates as a fallback). +func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string, branchCache *commitment.BranchCache) *SharedDomainsCommitmentContext { ctx := &SharedDomainsCommitmentContext{ sharedDomains: sd, tmpDir: tmpDir, } - ctx.patriciaTrie, ctx.updates = commitment.InitializeTrieAndUpdates(trieVariant, mode, tmpDir) + ctx.patriciaTrie, ctx.updates = commitment.InitializeTrieAndUpdates(trieVariant, mode, tmpDir, branchCache) return ctx } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 1ce1a1452ad..eede1637a5d 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2998,17 +2998,25 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { hph.leaveDeferredForCaller = leave } -// Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation +// Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation. +// +// The BranchCache is intentionally NOT cleared here: with aggregator-scope +// lifetime the cache must survive between commitment calculations to deliver +// cross-block hits. Callers that need to invalidate the cache (unwind, fork +// validation) MUST call ClearBranchCache explicitly. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false hph.rootChecked = false hph.rootPresent = true +} - // Clear the BranchCache only when this is the root trie (not a - // mounted subtrie). Mounted subtries share the root's cache; clearing - // from a mount would dump entries the root still expects to see. - // Carries the invariant established in PR #19954 commit 1612d5608c. +// ClearBranchCache drops every entry in the attached BranchCache. Use on +// unwind or fork-validation paths where the cached branches diverge from +// the canonical store. No-op when no cache is attached or when this is a +// mounted subtrie (the root trie owns the clear, mounts share the same +// cache pointer). +func (hph *HexPatriciaHashed) ClearBranchCache() { if hph.branchCache != nil && !hph.mounted { hph.branchCache.Clear() } From f87937e7389866de7cd4f53dc73afd184ab749c9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 11:11:00 +0000 Subject: [PATCH 006/120] commitment: BranchCache divergence detector + per-block fingerprint log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two debug-gated diagnostics for cross-block-cache-lifetime investigations. Both off by default — meant to be flipped on when a correctness regression surfaces and you need to localise *where in the fold path* the cache started lying, instead of waiting for a downstream trie-root-mismatch many blocks later. 1. BRANCH_CACHE_VERIFY: branchFromCacheOrDB cross-checks every L2 (BranchCache) hit against ctx.Branch and increments a divergence counter when bytes disagree. Logs the prefix and both byte forms so the first divergent read shows up directly in the erigon log. BranchCache.VerifyDivergences() exposes the count for assertion in tests. 2. BRANCH_CACHE_FINGERPRINT: SharedDomainsCommitmentContext.ComputeCommitment emits a "[cache-fp]" log at end of every compute with (block, root, cache fingerprint, divergence count). Two builds running the same workload can be diffed offline ("first block at which their fp's differ") to nail down the first block where lifecycle invariants diverged. Fingerprint is an order-independent FNV-1a fold over (key-hash, data-hash) pairs across both root and tail tiers. Adds maphash.LRU.Range so the Fingerprint can iterate the LRU tail without touching recency (Peek under the hood). The wrapper otherwise discards original byte-keys on insert; mixing by hash instead of key is correctness-equivalent up to hash collision, which is acceptable at the working-set sizes here. Motivation: today we just chased a wrong-root regression to commit d673052f (aggregator-scope cache lifetime). Took two full bench cycles (~12 min) to localise. With the divergence detector running, the first divergent read would surface in the erigon log within seconds. With fingerprint logging in two builds (one good, one regressed), the diverging block boundary would be a single grep across two log files. --- common/maphash/maphash.go | 21 ++++++ execution/commitment/branch_cache.go | 74 ++++++++++++++++++- .../commitmentdb/commitment_context.go | 24 ++++++ execution/commitment/hex_patricia_hashed.go | 21 ++++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/common/maphash/maphash.go b/common/maphash/maphash.go index 595e644f321..747fb45e1fd 100644 --- a/common/maphash/maphash.go +++ b/common/maphash/maphash.go @@ -163,3 +163,24 @@ func (l *LRU[V]) SetByHash(hash uint64, value V) { func (l *LRU[V]) ContainsByHash(hash uint64) bool { return l.cache.Contains(hash) } + +// Range iterates over every (hash, value) pair without affecting LRU +// recency (uses Peek under the hood). Iteration order is unspecified. +// Return false from fn to stop early. +// +// The original byte-key is not recoverable — Set hashes-and-discards. +// Use the hash itself as the identity in callers that need cross-cache +// comparisons; same byte-key always maps to the same hash, so equality +// of (hash, value) sets is equivalent to equality of (key, value) sets +// modulo collision (vanishingly unlikely at typical working-set sizes). +func (l *LRU[V]) Range(fn func(hash uint64, v V) bool) { + for _, h := range l.cache.Keys() { + v, ok := l.cache.Peek(h) + if !ok { + continue + } + if !fn(h, v) { + return + } + } +} diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 6497cfadb7f..388a852170f 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -137,6 +137,15 @@ type BranchCache struct { rootHits, rootMisses atomic.Uint64 tailHits, tailMisses atomic.Uint64 bytesServed atomic.Uint64 + + // Divergence counter — incremented by RecordDivergence when a caller + // detects that a cache-served value disagrees with the canonical + // store. Driven by branchFromCacheOrDB's verify path (gated by + // BRANCH_CACHE_VERIFY env). Helps localise correctness regressions + // in cross-block-cache investigations: a non-zero count is a + // load-bearing signal that the cache lifecycle is broken before any + // trie root mismatch surfaces downstream. + verifyDivergences atomic.Uint64 } type branchCacheEntry struct { @@ -332,6 +341,59 @@ func (c *BranchCache) Clear() { c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) + c.verifyDivergences.Store(0) +} + +// RecordDivergence increments the divergence counter. Called by +// branchFromCacheOrDB's verify path when a cache-served value disagrees +// with a parallel ctx.Branch read. +func (c *BranchCache) RecordDivergence() { + c.verifyDivergences.Add(1) +} + +// Fingerprint returns a deterministic hash of all current entries (root + +// tail). Two caches with the same set of (key, data) pairs produce the +// same fingerprint regardless of insertion order. Use for cross-run +// divergence localisation: emit per-block in two builds, diff the logs to +// see exactly which block their caches first differ. +// +// Mixes (key-hash, data-hash) pairs because the LRU stores by hash and +// discards the original key bytes on insert. Two entries with the same +// original key produce the same hash, so the fingerprint is still +// equality-equivalent to the (key, data) set modulo hash collision. +// +// Cheap: one FNV-1a fold over data per entry. Not cryptographic. +func (c *BranchCache) Fingerprint() uint64 { + const fnvOffset uint64 = 14695981039346656037 + const fnvPrime uint64 = 1099511628211 + dataHash := func(data []byte) uint64 { + h := fnvOffset + for _, b := range data { + h ^= uint64(b) + h *= fnvPrime + } + return h + } + mix := func(keyHash uint64, data []byte) uint64 { + // Combine key and data hashes into one entry hash; xor-fold across + // entries below so the per-cache result is insertion-order + // independent. + return keyHash ^ (dataHash(data) * fnvPrime) + } + var fp uint64 + if e := c.root.Load(); e != nil && len(e.data) > 0 { + // Pinned-root key is the constant 1-byte prefix 0x00; use a + // distinct sentinel hash so the root contribution can't collide + // with a tail entry hashed to zero. + fp ^= mix(0xdeadbeefcafe0001, e.data) + } + c.tail.Range(func(h uint64, e *branchCacheEntry) bool { + if len(e.data) > 0 { + fp ^= mix(h, e.data) + } + return true + }) + return fp } // Stats returns a one-line summary of root-tier and tail-tier hit/miss @@ -349,10 +411,20 @@ func (c *BranchCache) Stats() string { return 100.0 * float64(hit) / float64(total) } return fmt.Sprintf( - "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d", + "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d | divergences=%d", rh, rm, pct(rh, rm), th, tm, pct(th, tm), float64(bb)/1024/1024, c.tail.Len(), + c.verifyDivergences.Load(), ) } + +// VerifyDivergences returns the number of cache-vs-canonical divergences +// recorded since the last Clear. Non-zero indicates a cache lifecycle +// invariant has been violated (a cached entry no longer matches the +// canonical store) — read from outside to assert correctness in tests +// and benches. +func (c *BranchCache) VerifyDivergences() uint64 { + return c.verifyDivergences.Load() +} diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 2c59c686052..7d0cd5646a4 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -31,6 +31,13 @@ import ( var ( mxCommitmentRunning = metrics.GetOrCreateGauge("domain_running_commitment") mxCommitmentTook = metrics.GetOrCreateSummary("domain_commitment_took") + + // logCacheFingerprint, when true, makes ComputeCommitment emit a + // "[cache-fp]" log line at end-of-block with (block, root, fp, + // divergences). Diff two builds' logs offline to localise the first + // block at which their BranchCache states diverge. Off by default — + // gate via env BRANCH_CACHE_FINGERPRINT=true. + logCacheFingerprint = dbg.EnvBool("BRANCH_CACHE_FINGERPRINT", false) ) type sd interface { @@ -323,6 +330,23 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context keysPerSec = uint64(float64(updateCount) / took.Seconds()) } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) + + // Per-block BranchCache fingerprint — gated by BRANCH_CACHE_FINGERPRINT + // env. Emit a single log line of (block, root, cache_fp, divergences) + // so two builds running the same workload can be diffed offline to + // localise the first block at which their caches diverge. See + // commitment.BranchCache.Fingerprint for the hash semantics. + if logCacheFingerprint { + if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { + if bc := hph.BranchCache(); bc != nil { + log.Info("[commitment][cache-fp]", + "block", blockNum, + "root", hex.EncodeToString(rootHash), + "fp", fmt.Sprintf("%016x", bc.Fingerprint()), + "divergences", bc.VerifyDivergences()) + } + } + } }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index eede1637a5d..191379fc3e5 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3031,6 +3031,15 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { return hph.cache } +// verifyBranchCache, when true, makes branchFromCacheOrDB cross-check +// every BranchCache hit against ctx.Branch and record a divergence on +// the cache when the bytes disagree. Off by default — gate via env +// BRANCH_CACHE_VERIFY=true. Use during cross-block-cache-lifetime +// investigations to localise a bad cached entry to the read site +// instead of waiting for the downstream wrong-trie-root to surface +// many blocks later. +var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) + // branchFromCacheOrDB reads branch data via the cache hierarchy: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) // - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped @@ -3059,6 +3068,18 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { } if hph.branchCache != nil { if data, found := hph.branchCache.Get(key); found { + if verifyBranchCache { + canonical, _, err := hph.ctx.Branch(key) + if err == nil && !bytes.Equal(data, canonical) { + hph.branchCache.RecordDivergence() + log.Warn("[branch-cache] divergence", + "prefix", hex.EncodeToString(key), + "cached_len", len(data), + "canonical_len", len(canonical), + "cached", hex.EncodeToString(data), + "canonical", hex.EncodeToString(canonical)) + } + } return data, nil } } From b61655cadec16ea2eff1b85ada0dcba4bb0b86eb Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 17:42:45 +0000 Subject: [PATCH 007/120] commitment: disable deferred-encoding mode (use inline CollectUpdate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deferred-encoding path (CollectDeferredUpdate + ApplyDeferredUpdates) parallelises EncodeBranch + merge work at apply time. The cache correctness consequence: between collect and apply, sd.mem holds the old state while the cache is also unchanged. When apply finally fires (end of Process or duplicate-prefix flush mid-Process), sd.mem advances but the cache isn't touched — CollectDeferredUpdate doesn't have a cache hook (and wiring one breaks TestSharedDomain_RepeatedUnwindAcrossStepBoundary + TestCustomTraceReceiptDomain because it violates an implicit trie-during-Process cache stability invariant). The result on the FV bench: cache stays at the very-first-Process's read-side L3 fallback bytes for prefixes the trie writes, while ctx.Branch advances to each block's actual state. Divergence on every hot prefix (root, root-zone children) starting from block 2. Use CollectUpdate (inline) instead. CollectUpdate writes sd.mem + WarmupCache + BranchCache atomically at fold time via PutBranch — cache mirrors sd.mem at every write, the trie sees its own writes consistently, and the cross-Process cache state matches what post-FCU MDBX commit produced. Loses the encoder's parallel-encoding optimisation, but bench profile is I/O-bound, not encode-CPU-bound, so the trade is favourable. History (ETL) writes are still inline via DomainPut. Splitting that ("sd.mem inline, history queued for flush at FCU") is the proper architectural answer to defer the slow disk work without touching the sd.mem invariant — tracked as a follow-up. --- execution/commitment/hex_patricia_hashed.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 191379fc3e5..51d558bb6af 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -164,7 +164,14 @@ func newHexPatriciaHashed() *HexPatriciaHashed { } hph.branchEncoder.setMetrics(hph.metrics) - hph.branchEncoder.SetDeferUpdates(true) // Enable deferred branch updates by default + // Inline-mode write path: CollectUpdate writes sd.mem + WarmupCache + + // BranchCache atomically at fold-time via PutBranch, preserving the + // trie's read-your-own-writes invariant and keeping the cache aligned + // with sd.mem at every write. Deferred-mode (parallel encoding at + // apply time) breaks this — the encoder writes to sd.mem at apply + // time while reads still see pre-apply state via cache, producing + // intra-Process divergence. + hph.branchEncoder.SetDeferUpdates(false) return hph } From 7e21628c290874980dc9d009beadb25cb5df6aac Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 18:36:24 +0000 Subject: [PATCH 008/120] commitment: add DISABLE_BRANCH_CACHE_READS env kill-switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bisection helper: force branchFromCacheOrDB to skip the L2 BranchCache read path entirely so every read goes via ctx.Branch (sd.mem → MDBX). Cache writes still fire so verify-mode can keep comparing cache vs canonical. Flipping the env at runtime distinguishes "cache holds bad data" (bench passes further with cache reads off) from "deeper compute bug" (same failure regardless). Used in the 2026-05-06 investigation to confirm the cache was actively corrupting block 13's compute on the canonical SSTORE-bloat bench: with cache reads on, wrong-trie-root at block 13. With reads off, blocks 13-16 produce the correct roots and the bench advances to a separate failure at block 17 (unrelated pre-existing bug). Default off; gate via env DISABLE_BRANCH_CACHE_READS=true. --- execution/commitment/hex_patricia_hashed.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 51d558bb6af..cfbfc35df5c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3047,6 +3047,16 @@ func (hph *HexPatriciaHashed) Cache() *WarmupCache { // many blocks later. var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) +// disableBranchCacheReads forces branchFromCacheOrDB to skip the L2 +// BranchCache read path — every read goes to ctx.Branch (sd.mem → +// MDBX). Cache writes (CollectUpdate.Put, branchFromCacheOrDB.Put on +// L3 fallback) still fire so verify-mode can keep comparing cache vs +// canonical. Use to A/B test correctness with-cache vs without-cache +// without recompiling. Confirmed in 2026-05-06 investigation that +// flipping this distinguishes "cache holds bad data" (bench passes +// further) from "deeper compute bug" (bench fails at the same block). +var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) + // branchFromCacheOrDB reads branch data via the cache hierarchy: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) // - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped @@ -3073,7 +3083,7 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } - if hph.branchCache != nil { + if hph.branchCache != nil && !disableBranchCacheReads { if data, found := hph.branchCache.Get(key); found { if verifyBranchCache { canonical, _, err := hph.ctx.Branch(key) From 8d6d19a3aff9162b65d70c70705e9dd51af9d445 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 08:38:07 +0000 Subject: [PATCH 009/120] commitment: multi-layer state probe at BranchCache divergence site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When verifyBranchCache=true and a cache hit disagrees with ctx.Branch, sample sd.mem, sd.parent.mem, and tx-direct (MDBX) for the same prefix and dump all layers in the divergence log line. Comparing those four byte sequences against the cached and canonical bytes pinpoints which state layer holds the bytes the cache disagrees with — the rewriter we need to identify before fixing the canonical-store-divergence bugs the cache currently exposes. Decision matrix (read off the log line): - cache != tx, sd.mem == cache → in-memory writer is fresh, MDBX is stale (commit-timing issue). - cache != tx, tx == ctx.Branch, sd.mem != cache, parent.mem != cache → MDBX has been rewritten by something outside the CollectUpdate write path (collation, file build, squeeze). - cache != ctx.Branch, parent.mem matches ctx.Branch but sd.mem doesn't → parent merge is the source. - cache != ctx.Branch, all of sd.mem / parent.mem / tx == ctx.Branch → cache itself was populated incorrectly (write-side bug). Three changes: 1. SharedDomains.ProbeReadLayers (db/state/execctx/domain_shared.go): public method that samples sd.mem, sd.parent.mem (private field accessed from the same package), and tx.GetLatest. Read-only; copies bytes so callers can hold them past tx lifetime. 2. TrieContext (execution/commitment/commitmentdb/commitment_context.go): add probeSd + probeTx fields populated at trieContext() construction; expose ProbeStateLayers method that delegates to sd.ProbeReadLayers. The local `sd` interface gets the ProbeReadLayers method too so the duck-typed reference can call it without an import cycle to execctx. 3. branchFromCacheOrDB log site (execution/commitment/hex_patricia_hashed.go): on divergence with verifyBranchCache, type-assert ctx for the probe interface and append sd_mem / parent_mem / mdbx fields to the log line. Existing field shape preserved for log parsers; new fields are additive. Pre-existing test failures (TestSharedDomain_RepeatedUnwindAcrossStepBoundary, TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeight) are unchanged — they were failing on the stack before this probe landed and are part of what the divergence work is meant to localise. --- db/state/execctx/domain_shared.go | 25 +++++++++++++++++ .../commitmentdb/commitment_context.go | 28 +++++++++++++++++++ execution/commitment/hex_patricia_hashed.go | 28 +++++++++++++++++-- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 71d206f5de6..9469c447667 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -580,6 +580,31 @@ func (sd *SharedDomains) GetStateCache() *cache.StateCache { return sd.stateCache } +// ProbeReadLayers samples each independent state layer (this SD's mem, +// the parent SD's mem, and direct MDBX via tx.GetLatest) and returns +// the bytes from each. Read-only, intended for divergence-detection +// diagnostics — pinpoints which layer holds bytes that disagree with +// the BranchCache when verifyBranchCache is on. Bytes are copied so +// the caller can hold them past tx lifetime. +func (sd *SharedDomains) ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { + if v, _, ok := sd.mem.GetLatest(domain, key); ok { + memOk = true + mem = append([]byte(nil), v...) + } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(domain, key); ok { + parentOk = true + parentMem = append([]byte(nil), v...) + } + } + if tx != nil { + if v, _, err := tx.GetLatest(domain, key); err == nil { + mdbx = append([]byte(nil), v...) + } + } + return +} + func (sd *SharedDomains) ClearRam(resetCommitment bool) { // When the commitment calculator goroutine owns the Updates buffer, // skip ClearRam on the commitment context to avoid concurrent btree access. diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 7d0cd5646a4..94769f54bce 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -47,6 +47,12 @@ type sd interface { StepSize() uint64 Trace() bool CommitmentCapture() bool + + // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) + // for one key. Used by the BranchCache divergence-detection probe + // to localise which state layer holds bytes that diverge from + // cache. Read-only. + ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) } type SharedDomainsCommitmentContext struct { @@ -217,6 +223,8 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu stepSize: sdc.sharedDomains.StepSize(), txNum: txNum, blockNum: blockNum, + probeSd: sdc.sharedDomains, + probeTx: tx, } if sdc.stateReader != nil { mainTtx.stateReader = sdc.stateReader.Clone(tx) @@ -817,6 +825,14 @@ type TrieContext struct { trace bool stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch + + // probeSd / probeTx feed ProbeStateLayers — diagnostics-only fields + // populated when this TrieContext is constructed from a + // SharedDomainsCommitmentContext that has both a SharedDomains + // and a tx in scope. Both nil for read-only / test contexts where + // the probe is not available. + probeSd sd + probeTx kv.TemporalTx } // NewTrieContextRo creates a read-only TrieContext suitable for TrieReader lookups. @@ -838,6 +854,18 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } +// ProbeStateLayers samples the underlying state layers for one key — +// sd.mem, sd.parent.mem (if any), and tx-direct (MDBX) — for +// divergence-detection diagnostics. Returns empty / not-ok when +// constructed without a probe-capable SharedDomains (e.g. +// NewTrieContextRo). Read-only. +func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { + if sdc.probeSd == nil { + return + } + return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) +} + func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index cfbfc35df5c..c8766e7cf6c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -40,6 +40,7 @@ import ( "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/stateifs" "github.com/erigontech/erigon/execution/commitment/nibbles" "github.com/erigontech/erigon/execution/commitment/trie" @@ -3089,12 +3090,35 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { canonical, _, err := hph.ctx.Branch(key) if err == nil && !bytes.Equal(data, canonical) { hph.branchCache.RecordDivergence() - log.Warn("[branch-cache] divergence", + fields := []any{ "prefix", hex.EncodeToString(key), "cached_len", len(data), "canonical_len", len(canonical), "cached", hex.EncodeToString(data), - "canonical", hex.EncodeToString(canonical)) + "canonical", hex.EncodeToString(canonical), + } + // State-layer probe: sample sd.mem, parent.mem, and + // tx-direct (MDBX) for the same key so the log line + // shows which layer holds bytes matching the cache + // vs ctx.Branch. Decision matrix in + // agentspecs/branch-cache-divergence-probe.md. + if probe, ok := hph.ctx.(interface { + ProbeStateLayers(kv.Domain, []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) + }); ok { + mem, parentMem, mdbx, memOk, parentOk := probe.ProbeStateLayers(kv.CommitmentDomain, key) + if memOk { + fields = append(fields, "sd_mem", hex.EncodeToString(mem)) + } else { + fields = append(fields, "sd_mem", "miss") + } + if parentOk { + fields = append(fields, "parent_mem", hex.EncodeToString(parentMem)) + } else { + fields = append(fields, "parent_mem", "miss-or-no-parent") + } + fields = append(fields, "mdbx", hex.EncodeToString(mdbx)) + } + log.Warn("[branch-cache] divergence", fields...) } } return data, nil From 87f12be64407eb223c33b3f32b4a4a7ab116c1b8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 10:59:17 +0000 Subject: [PATCH 010/120] commitment: tag BranchCache writes with origin + write-seq + timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-write provenance for divergence-detection diagnostics. When a divergence fires (cache hit disagrees with ctx.Branch), we now log which write site produced the cached bytes and when, so we can correlate the bad write against the FCU / build / step timeline. Tag fields added to branchCacheEntry: - origin short label of the write site (e.g. "CollectUpdate", "L3-fallback-read") - writeSeq monotonic counter per BranchCache instance - writeTimeNanos unix nanos at write time Put / PutIfClean signatures take an origin string. Two writers updated: - BranchEncoder.CollectUpdate → "CollectUpdate" - branchFromCacheOrDB L3-fallback Put → "L3-fallback-read" GetWithOrigin returns bytes plus the metadata; uses a non-counting peek so it can be called alongside Get without double-counting hits. The divergence-detection log site at branchFromCacheOrDB now appends cache_origin / cache_seq / cache_t_ns fields. Combine with the existing sd_mem / parent_mem / mdbx fields to localise both who wrote the stale bytes and which layer the canonical value lives in. Pre-existing test failures (TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeight) are unchanged from previous commits. --- execution/commitment/branch_cache.go | 54 +++++++++++++++++++-- execution/commitment/branch_cache_test.go | 30 ++++++------ execution/commitment/commitment.go | 2 +- execution/commitment/hex_patricia_hashed.go | 14 +++++- 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 388a852170f..4adf87b0609 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -20,6 +20,7 @@ import ( "fmt" "sync" "sync/atomic" + "time" "github.com/erigontech/erigon/common/maphash" ) @@ -146,6 +147,12 @@ type BranchCache struct { // load-bearing signal that the cache lifecycle is broken before any // trie root mismatch surfaces downstream. verifyDivergences atomic.Uint64 + + // writeSeq is incremented on every Put so each entry carries a + // monotonic ordering tag — divergence-detection uses this with the + // origin label and timestamp to identify which write produced the + // stale bytes. + writeSeq atomic.Uint64 } type branchCacheEntry struct { @@ -166,6 +173,15 @@ type branchCacheEntry struct { // entry was populated; treat as stale until cleared." Same semantics // as branchEntry.dirty in WarmupCache (carried from step 4). dirty atomic.Bool + + // origin diagnostics — captured at Put time so divergence-detection + // can identify which write produced the (now-disagreeing) bytes. + // origin is a short label of the write site (e.g. "CollectUpdate", + // "L3-fallback-read"); writeSeq is a monotonic counter per + // BranchCache instance; writeTimeNanos is unix-nanos at write time. + origin string + writeSeq uint64 + writeTimeNanos int64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -284,11 +300,17 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. -func (c *BranchCache) Put(prefix []byte, data []byte) { +// caller buffer lifetime. origin is a short label of the write site +// captured for divergence-detection diagnostics. +func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { dataCopy := make([]byte, len(data)) copy(dataCopy, data) - c.store(prefix, &branchCacheEntry{data: dataCopy}) + c.store(prefix, &branchCacheEntry{ + data: dataCopy, + origin: origin, + writeSeq: c.writeSeq.Add(1), + writeTimeNanos: time.Now().UnixNano(), + }) } // PutIfClean stores branch data only if no existing entry is marked dirty. @@ -298,14 +320,36 @@ func (c *BranchCache) Put(prefix []byte, data []byte) { // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data) + c.Put(prefix, data, origin) return true } +// GetWithOrigin returns the cached bytes plus the diagnostic origin +// metadata captured when the entry was put. ok=false on miss. Used by +// the divergence-detection probe to identify which write produced the +// stale bytes. Does not bump hit/miss counters or affect LRU recency +// (uses a non-counting peek so it can be called alongside Get without +// double-counting). +func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, writeSeq uint64, writeTimeNanos int64, ok bool) { + var entry *branchCacheEntry + if isRootPrefix(prefix) { + entry = c.root.Load() + } else { + entry, ok = c.tail.Get(prefix) + if !ok { + return nil, "", 0, 0, false + } + } + if entry == nil { + return nil, "", 0, 0, false + } + return entry.data, entry.origin, entry.writeSeq, entry.writeTimeNanos, true +} + // MarkDirty flags the entry at prefix as stale-until-cleared. Subsequent // PutIfClean calls for this prefix will skip; reads still return the // entry (the dirty signal is consumed only on the write path today, same diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 5338a090701..3340e5cb292 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,8 +31,8 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data")) - c.Put(deepKey, []byte("deep-data")) + c.Put(rootKey, []byte("root-data"), "test") + c.Put(deepKey, []byte("deep-data"), "test") // Root reads should increment rootHits, not tailHits got, ok := c.Get(rootKey) @@ -55,11 +55,11 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS")) + c.Put(rootKey, []byte("ROOT-PERSISTS"), "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}) + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, "test") } // Root must still be there @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"))) + require.True(t, c.PutIfClean(key, []byte("v1"), "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2")), "PutIfClean must refuse dirty entry") + require.False(t, c.PutIfClean(key, []byte("v2"), "test"), "PutIfClean must refuse dirty entry") got, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3")) + c.Put(key, []byte("v3"), "test") got, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"))) + require.True(t, c.PutIfClean(key, []byte("v4"), "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc) + c.Put(prefix, enc, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -130,8 +130,8 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r")) - c.Put(deepKey, []byte("d")) + c.Put(rootKey, []byte("r"), "test") + c.Put(deepKey, []byte("d"), "test") c.Invalidate(rootKey) _, ok := c.Get(rootKey) @@ -145,8 +145,8 @@ func TestBranchCache_Invalidate(t *testing.T) { // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r")) - c.Put([]byte{0x12}, []byte("d")) + c.Put([]byte{0x00}, []byte("r"), "test") + c.Put([]byte{0x12}, []byte("d"), "test") c.Get([]byte{0x00}) c.Get([]byte{0x12}) @@ -166,8 +166,8 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr")) - c.Put([]byte{0x12, 0x34}, []byte("ddd")) + c.Put([]byte{0x00}, []byte("rrr"), "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), "test") c.Get([]byte{0x00}) c.Get([]byte{0x12, 0x34}) c.Get([]byte{0xff}) // tail miss diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ddedeb95010..8a4cce06390 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -641,7 +641,7 @@ func (be *BranchEncoder) CollectUpdate( // (the new entry is born clean). Single writer per prefix per fold // invariant means no concurrent Put races on this key. if be.branchCache != nil { - be.branchCache.Put(prefixCopy, updateCopy) + be.branchCache.Put(prefixCopy, updateCopy, "CollectUpdate") } if be.metrics != nil { be.metrics.updateBranch.Add(1) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index c8766e7cf6c..71ed6dc909c 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -3097,6 +3097,18 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { "cached", hex.EncodeToString(data), "canonical", hex.EncodeToString(canonical), } + // Origin metadata captured at cache.Put time — + // identifies which write site produced the stale + // bytes (CollectUpdate vs L3-fallback-read), with + // a monotonic write-seq + unix-nanos timestamp so + // we can correlate against the timeline of FCU / + // build / step events. + if _, origin, seq, tns, ok := hph.branchCache.GetWithOrigin(key); ok { + fields = append(fields, + "cache_origin", origin, + "cache_seq", seq, + "cache_t_ns", tns) + } // State-layer probe: sample sd.mem, parent.mem, and // tx-direct (MDBX) for the same key so the log line // shows which layer holds bytes matching the cache @@ -3129,7 +3141,7 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { return nil, err } if hph.branchCache != nil && len(data) > 0 { - hph.branchCache.Put(key, data) + hph.branchCache.Put(key, data, "L3-fallback-read") } return data, nil } From 6989427ad99b838de488c7e3960f734a5323c009 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 12:14:02 +0000 Subject: [PATCH 011/120] commitment, db/state/execctx: move BranchCache behind sd.mem chain BranchCache previously sat in front of the sd.mem -> parent.mem -> MDBX read chain (consulted in branchFromCacheOrDB before ctx.Branch). The shared aggregator-scope cache was written from CollectUpdate by every SD running commitment compute, including fork-validator SDs whose writes never reach MDBX. Origin-tagged probe (run-step7b-probe-sdid) showed five distinct SD pointers writing the same prefix to a single cache entry, so any reader whose lineage didn't match the most-recent writer saw bytes that disagreed with MDBX -> wrong trie root from block 13 onward in the canonical SSTORE-bloat fork bench. Layering after this change: Read: sd.mem -> sd.parent.mem -> branchCache -> aggTx (MDBX) Write: sd.mem only (DomainPut path) Flush: sd.mem -> MDBX, then branchCache.Clear() The cache now mirrors MDBX-flushed bytes only. Writers' in-flight bytes live in sd.mem above; cache hits below sd.mem are always equivalent to reading MDBX, so cross-SD pollution is impossible by construction. Cache fills lazily on the MDBX-read path inside sd.GetLatest, and clear-on-flush prevents pre-flush bytes from coexisting with new MDBX state. Per-key invalidation is a follow-up (PR2). BranchCache entries gain a step field so Get returns (data, step, ok) matching the aggTx contract. Without this, sd.GetLatest's cache hit returned step=0 and CheckDataAvailable rejected the boot SeekCommitment with "commitment state out of date". Removed: - cache.Put from CollectUpdate (commitment.go) - cache.Put + divergence detection from branchFromCacheOrDB (hex_patricia_hashed.go); now just calls ctx.Branch - L3-fallback Put (cache fills via sd.GetLatest now) Validated on canonical cold bench (run-step8b): first FCU VALID, all payloads through end VALID, 0 cache divergences, 0 wrong-root errors. Prior probe bench had 23 divergences and INVALID payloads from the fail block onward. Probe scaffolding (SiteIdentity, ProbeStateLayers, divergence counters) left in place for now; can be stripped in a cleanup follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 35 ++++++++- execution/commitment/branch_cache.go | 26 +++++-- execution/commitment/branch_cache_test.go | 60 +++++++-------- execution/commitment/commitment.go | 6 +- .../commitmentdb/commitment_context.go | 8 ++ execution/commitment/hex_patricia_hashed.go | 74 ++----------------- 6 files changed, 99 insertions(+), 110 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 9469c447667..7eb25fe20f1 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -158,6 +158,14 @@ type SharedDomains struct { // (now tx-granular) at sd.Flush time, and delete this Mutex + the // SetChangesetAccumulator/GetChangesetAccumulator API entirely. changesetMu sync.Mutex + + // branchCache is the aggregator-scope commitment-branch cache. It sits + // behind sd.mem and sd.parent.mem in the read chain (consulted only after + // both miss, before the aggTx files/MDBX read), so writers' in-flight + // bytes always mask the cache and cross-SD pollution is impossible. + // May be nil for test setups whose AggTx doesn't implement + // commitment.BranchCacheProvider. + branchCache *commitment.BranchCache } func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger) (*SharedDomains, error) { @@ -190,6 +198,7 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } + sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) _, blockNum, err := sd.SeekCommitment(ctx, tx) @@ -695,7 +704,17 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - return sd.mem.Flush(ctx, tx) + if err := sd.mem.Flush(ctx, tx); err != nil { + return err + } + // Cache mirrors MDBX-flushed bytes; clear after flush so cache cannot + // hold pre-flush bytes for keys whose MDBX value just changed. Refills + // lazily on next read. PR2 will switch to per-key invalidation using + // the just-flushed CommitmentDomain diff. + if sd.branchCache != nil { + sd.branchCache.Clear() + } + return nil } // TemporalDomain satisfaction @@ -764,6 +783,17 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } + // branchCache sits between sd.mem/parent.mem and the aggTx files for + // CommitmentDomain only. It mirrors MDBX-flushed bytes; writers' fresh + // bytes are held in sd.mem above, so a cache hit here is always + // equivalent to reading from MDBX. Cleared by sd.Flush so a new MDBX + // state never coexists with stale cache entries. + if domain == kv.CommitmentDomain && sd.branchCache != nil { + if cv, cstep, ok := sd.branchCache.Get(k); ok { + return cv, kv.Step(cstep), nil + } + } + if aggTx, ok := tx.AggTx().(MeteredGetter); ok { v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) } else { @@ -777,6 +807,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) if sd.stateCache != nil { sd.stateCache.Put(domain, k, v) } + if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { + sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + } return v, step, nil } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 4adf87b0609..a24f1193172 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -182,6 +182,13 @@ type branchCacheEntry struct { origin string writeSeq uint64 writeTimeNanos int64 + + // step is the on-disk file step the cached bytes came from. Returned + // by Get so callers (e.g. CheckDataAvailable) can validate against + // the latest visible step. 0 means "step not tracked" — fine for + // in-memory tests but real callers should always pass the step + // returned by aggTx.MeteredGetLatest / tx.GetLatest. + step uint64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -251,14 +258,15 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { } // Get retrieves branch data from the cache. Returns the canonical encoded -// bytes (with the leading 2-byte touch-map prefix). -func (c *BranchCache) Get(prefix []byte) ([]byte, bool) { +// bytes (with the leading 2-byte touch-map prefix) plus the on-disk file +// step the bytes came from (0 if not tracked). +func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { entry, ok := c.lookup(prefix) if !ok { - return nil, false + return nil, 0, false } c.bytesServed.Add(uint64(len(entry.data))) - return entry.data, true + return entry.data, entry.step, true } // GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on @@ -300,13 +308,15 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. origin is a short label of the write site +// caller buffer lifetime. step is the on-disk file step the bytes came +// from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { +func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ data: dataCopy, + step: step, origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -320,11 +330,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, origin string) { // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, origin) + c.Put(prefix, data, step, origin) return true } diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 3340e5cb292..f545a23a650 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,18 +31,18 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data"), "test") - c.Put(deepKey, []byte("deep-data"), "test") + c.Put(rootKey, []byte("root-data"), 0, "test") + c.Put(deepKey, []byte("deep-data"), 0, "test") // Root reads should increment rootHits, not tailHits - got, ok := c.Get(rootKey) + got, _, ok := c.Get(rootKey) require.True(t, ok) require.Equal(t, []byte("root-data"), got) require.Equal(t, uint64(1), c.rootHits.Load()) require.Equal(t, uint64(0), c.tailHits.Load()) // Deep reads should increment tailHits, not rootHits - got, ok = c.Get(deepKey) + got, _, ok = c.Get(deepKey) require.True(t, ok) require.Equal(t, []byte("deep-data"), got) require.Equal(t, uint64(1), c.rootHits.Load()) @@ -55,15 +55,15 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS"), "test") + c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, "test") + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, "test") } // Root must still be there - got, ok := c.Get(rootKey) + got, _, ok := c.Get(rootKey) require.True(t, ok, "root should never be evicted from pinned slot") require.Equal(t, []byte("ROOT-PERSISTS"), got) @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"), "test")) + require.True(t, c.PutIfClean(key, []byte("v1"), 0, "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2"), "test"), "PutIfClean must refuse dirty entry") - got, _ := c.Get(key) + require.False(t, c.PutIfClean(key, []byte("v2"), 0, "test"), "PutIfClean must refuse dirty entry") + got, _, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3"), "test") - got, _ = c.Get(key) + c.Put(key, []byte("v3"), 0, "test") + got, _, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"), "test")) + require.True(t, c.PutIfClean(key, []byte("v4"), 0, "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc, "test") + c.Put(prefix, enc, 0, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -120,7 +120,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.Same(t, cells, cells2, "decoded form cached and reused") // Encoded form unchanged after decoded reads - encGot, ok := c.Get(prefix) + encGot, _, ok := c.Get(prefix) require.True(t, ok) require.Equal(t, []byte(enc), encGot) } @@ -130,25 +130,25 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r"), "test") - c.Put(deepKey, []byte("d"), "test") + c.Put(rootKey, []byte("r"), 0, "test") + c.Put(deepKey, []byte("d"), 0, "test") c.Invalidate(rootKey) - _, ok := c.Get(rootKey) + _, _, ok := c.Get(rootKey) require.False(t, ok, "root invalidated") c.Invalidate(deepKey) - _, ok = c.Get(deepKey) + _, _, ok = c.Get(deepKey) require.False(t, ok, "deep invalidated") } // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r"), "test") - c.Put([]byte{0x12}, []byte("d"), "test") - c.Get([]byte{0x00}) - c.Get([]byte{0x12}) + c.Put([]byte{0x00}, []byte("r"), 0, "test") + c.Put([]byte{0x12}, []byte("d"), 0, "test") + _, _, _ = c.Get([]byte{0x00}) + _, _, _ = c.Get([]byte{0x12}) require.Equal(t, uint64(1), c.rootHits.Load()) require.Equal(t, uint64(1), c.tailHits.Load()) @@ -156,9 +156,9 @@ func TestBranchCache_Clear(t *testing.T) { c.Clear() require.Equal(t, uint64(0), c.rootHits.Load()) require.Equal(t, uint64(0), c.tailHits.Load()) - _, ok := c.Get([]byte{0x00}) + _, _, ok := c.Get([]byte{0x00}) require.False(t, ok) - _, ok = c.Get([]byte{0x12}) + _, _, ok = c.Get([]byte{0x12}) require.False(t, ok) } @@ -166,11 +166,11 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr"), "test") - c.Put([]byte{0x12, 0x34}, []byte("ddd"), "test") - c.Get([]byte{0x00}) - c.Get([]byte{0x12, 0x34}) - c.Get([]byte{0xff}) // tail miss + c.Put([]byte{0x00}, []byte("rrr"), 0, "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, "test") + _, _, _ = c.Get([]byte{0x00}) + _, _, _ = c.Get([]byte{0x12, 0x34}) + _, _, _ = c.Get([]byte{0xff}) // tail miss s := c.Stats() for _, want := range []string{ diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 8a4cce06390..5fb10080079 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -640,9 +640,9 @@ func (be *BranchEncoder) CollectUpdate( // fresh canonical bytes — clears the dirty flag in the process // (the new entry is born clean). Single writer per prefix per fold // invariant means no concurrent Put races on this key. - if be.branchCache != nil { - be.branchCache.Put(prefixCopy, updateCopy, "CollectUpdate") - } + // Cache is populated by sd.GetLatest on MDBX reads; CollectUpdate writes + // only to sd.mem (via ctx.PutBranch above). Pre-flush invalidation isn't + // needed because sd.mem masks the cache for any prefix the writer touched. if be.metrics != nil { be.metrics.updateBranch.Add(1) } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 94769f54bce..88fab39952c 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -866,6 +866,14 @@ func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, par return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) } +// SiteIdentity returns a stable string identifying the underlying +// SharedDomains pointer. Used by cache write sites to tag entries with +// the SD lineage that produced them, so divergence diagnostics can tell +// parent-SD writes apart from fork-SD writes when the cache is shared. +func (sdc *TrieContext) SiteIdentity() string { + return fmt.Sprintf("sd=%p", sdc.probeSd) +} + func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 71ed6dc909c..2f86a61a6d7 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -40,7 +40,6 @@ import ( "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/stateifs" "github.com/erigontech/erigon/execution/commitment/nibbles" "github.com/erigontech/erigon/execution/commitment/trie" @@ -3058,20 +3057,14 @@ var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) // further) from "deeper compute bug" (bench fails at the same block). var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) -// branchFromCacheOrDB reads branch data via the cache hierarchy: +// branchFromCacheOrDB reads branch data through the SD chain: // - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) -// - L2: BranchCache (longer-lived; per-Process today, will be aggTx-scoped -// once the cross-block-persistence step lands) -// - L3: ctx.Branch (canonical store) +// - L2: ctx.Branch — sd.mem -> sd.parent.mem -> branchCache -> MDBX // -// On L2 hit no further work happens. On L3 read with a non-empty result the -// bytes are populated into BranchCache so subsequent reads (within the cache's -// lifetime) hit L2 instead. -// -// L1 (WarmupCache) is intentionally checked FIRST: warmup workers may have -// pre-fetched the branch with prefix-walk-derived freshness guarantees -// (see warmuper.go), and L1 entries are short-lived enough that staleness -// is bounded by Process duration. +// The aggregator-scope BranchCache lives behind sd.mem inside sd.GetLatest; +// CollectUpdate writes only to sd.mem, so cross-SD pollution can no longer +// occur. L1 (WarmupCache) is still checked first for prefix-walk-derived +// freshness within a Process. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { if hph.cache != nil { if data, found := hph.cache.GetBranch(key); found { @@ -3084,65 +3077,10 @@ func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { hph.metrics.missBranch.Add(1) } } - if hph.branchCache != nil && !disableBranchCacheReads { - if data, found := hph.branchCache.Get(key); found { - if verifyBranchCache { - canonical, _, err := hph.ctx.Branch(key) - if err == nil && !bytes.Equal(data, canonical) { - hph.branchCache.RecordDivergence() - fields := []any{ - "prefix", hex.EncodeToString(key), - "cached_len", len(data), - "canonical_len", len(canonical), - "cached", hex.EncodeToString(data), - "canonical", hex.EncodeToString(canonical), - } - // Origin metadata captured at cache.Put time — - // identifies which write site produced the stale - // bytes (CollectUpdate vs L3-fallback-read), with - // a monotonic write-seq + unix-nanos timestamp so - // we can correlate against the timeline of FCU / - // build / step events. - if _, origin, seq, tns, ok := hph.branchCache.GetWithOrigin(key); ok { - fields = append(fields, - "cache_origin", origin, - "cache_seq", seq, - "cache_t_ns", tns) - } - // State-layer probe: sample sd.mem, parent.mem, and - // tx-direct (MDBX) for the same key so the log line - // shows which layer holds bytes matching the cache - // vs ctx.Branch. Decision matrix in - // agentspecs/branch-cache-divergence-probe.md. - if probe, ok := hph.ctx.(interface { - ProbeStateLayers(kv.Domain, []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) - }); ok { - mem, parentMem, mdbx, memOk, parentOk := probe.ProbeStateLayers(kv.CommitmentDomain, key) - if memOk { - fields = append(fields, "sd_mem", hex.EncodeToString(mem)) - } else { - fields = append(fields, "sd_mem", "miss") - } - if parentOk { - fields = append(fields, "parent_mem", hex.EncodeToString(parentMem)) - } else { - fields = append(fields, "parent_mem", "miss-or-no-parent") - } - fields = append(fields, "mdbx", hex.EncodeToString(mdbx)) - } - log.Warn("[branch-cache] divergence", fields...) - } - } - return data, nil - } - } data, _, err := hph.ctx.Branch(key) if err != nil { return nil, err } - if hph.branchCache != nil && len(data) > 0 { - hph.branchCache.Put(key, data, "L3-fallback-read") - } return data, nil } From 99823bbe9e931cf66f8004dd599f9be619253c7a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:49:38 +0000 Subject: [PATCH 012/120] commitment/branch_cache: document responsibility split + post-WarmupCache state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the BranchCache type comment to reflect the architectural state after the WarmupCache consolidation (steps 2a-2c, 3, 4): - BranchCache is the single branch cache (WarmupCache deleted) - Aggregator-scope lifetime, plumbed via BranchCacheProvider - Passive store: cache itself never reaches into underlying state - Branch warmer is branch-scoped; no leaf-data prefetch - Block-processing trie walker takes Updates from executor + memoization for siblings — no prefetch needed - Witness / proof generation walker drives its own state reads; if that path turns out to be cold-bound it indicates a need for separate account/storage caches (treat as separate concern with different scope/lifetime/invalidation; do not regrow the branch warmer to cover it) - disk_sto / disk_acc counters on cache-fp log surface any unexpected fall-through to ctx.Account / ctx.Storage as a signal of memoization gap or missing walker-side prefetch Doc-only; no behaviour change. --- execution/commitment/branch_cache.go | 62 ++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index a24f1193172..bedc7d9db53 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -25,8 +25,7 @@ import ( "github.com/erigontech/erigon/common/maphash" ) -// BranchCache stores commitment-trie branch data with a persistence-friendly -// shape distinct from WarmupCache: +// BranchCache stores commitment-trie branch data: // // - Bounded LRU tail with configurable capacity (eviction is well-defined, // suitable for long-lived caching across many Process calls without @@ -34,24 +33,53 @@ import ( // - Single pinned slot for the root branch (always hottest, always present // once populated, never subject to LRU eviction). Compact prefix of // length 0 (or single-byte "no-key" form) targets this slot. -// - dirty-flag + PutIfClean invariants (carried from step 4 of the -// representation-reduction sequence) so cross-block writers can race +// - dirty-flag + PutIfClean invariants so cross-block writers can race // safely with fold updates. -// - Lazy-decoded read path (GetDecoded) — same pattern as WarmupCache's -// GetBranchDecoded; cells are populated from the cached encoded form -// on first decoded-read and reused thereafter. +// - Lazy-decoded read path (GetDecoded) — cells are populated from the +// cached encoded form on first decoded-read and reused thereafter. // -// Today this is constructed as ephemeral infrastructure (per-Process, -// alongside the trie). Future cross-block persistence work (step 7 of -// the representation-reduction sequence) will lift its lifetime to the -// aggTx level. The wire-up between the trie's read path and BranchCache -// is intentionally NOT done in this commit — see the integration -// discussion captured at the time of step 6. +// Lifetime: aggregator-scope (one instance per Domain). SharedDomains +// pulls the instance via BranchCacheProvider on the AggregatorRoTx; +// commitment-context plumbs it through to the trie via +// InitializeTrieAndUpdates. The previous WarmupCache type (per-Process, +// duplicating account/storage/branch caching above this layer) was +// deleted in the WarmupCache consolidation; BranchCache is now the +// single branch cache. // -// Distinct from WarmupCache: WarmupCache is a simple unbounded map for -// warmup-fill hot-read patterns within one Process. BranchCache is -// designed for the longer-lived cache lifetime that the cross-block -// persistence work needs. +// # Responsibility split (architectural) +// +// The cache is a passive store. Reads and writes are driven by the +// trie walker / encoder; the cache itself never reaches into the +// underlying state. +// +// - BranchCache: passive store of branch bytes / decoded cells. +// Doesn't fetch anything. +// - Branch warmer (warmuper.go): narrow scope — pre-fetches +// *branches* along touched-key paths via SD.GetLatest. No +// account/storage prefetch — that conflated branch warm-up with +// leaf-data fetch. If a fold needs leaf data the trie walker +// fetches it directly (or it's already in Updates / memoized as +// stateHash). +// - Trie walker, block-processing path: receives Updates from the +// executor, folds them. Memoized stateHashes serve siblings; new +// values come from Updates. Doesn't reach into leaf data via +// prefetch. +// - Trie walker, witness / proof generation path: walks the trie +// structure and *needs* to fetch state to materialize the proof. +// This is the walker's responsibility — it drives its own reads +// against SD. If that path turns out to be cold-bound on real +// workloads it may indicate a need for separate account / storage +// caches (the `add_execution_context_with_caches` work has a +// reference design for these). Treat that as a separate concern +// from this BranchCache — different scope, different lifetime, +// different invalidation. Do not regrow the branch warmer's +// scope to cover it. +// +// The disk_sto / disk_acc counters on the [commitment][cache-fp] log +// line surface any fall-through where the trie compute reaches the +// underlying ctx.Account / ctx.Storage paths. On block-processing +// workloads they should remain zero; non-zero values signal a +// memoization gap or a missing walker-side prefetch. // // # Concurrency contract — caller invariants // From 4410704e7a65615be2c9d803a391ff52b54d6dc5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:57:28 +0000 Subject: [PATCH 013/120] commitment/branch_cache: add pinned tier (PinEntry / PinnedCount) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third cache tier between the root-pin slot and the LRU tail: a per-prefix pinned map fed by PinEntry. Pinned entries: - Never evict (no LRU pressure on this tier). - Are checked in lookup before the LRU tail, after the root slot. - Survive Put/SD.Flush updates: a Put for a pinned prefix updates the pinned entry in place rather than displacing it. Cross-block correctness via the existing dirty-flag invalidation discipline is preserved — the new bytes land in the same pinned slot. - Carry the same metadata as Put-tier entries (step, origin, writeSeq, writeTimeNanos) so divergence-detection and Stats treat them uniformly. Sized by the preload policy. Intended consumer is the storage trunk preload for big contracts (the 'storage root trunk cache for big accounts' direction): per-contract trunk branches at depth 65-70 get pinned at SD/cache creation, persist for the cache's lifetime. PinEntry is the public API; pinnedHits/pinnedMisses atomic counters are the new stats. PinnedCount() exposes current size for observability. Step 2 of the storage-trunk pin prototype. --- execution/commitment/branch_cache.go | 62 ++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index bedc7d9db53..a7a64c59b4a 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -158,14 +158,24 @@ type BranchCache struct { // access so no lock is needed for the hot read path. root atomic.Pointer[branchCacheEntry] + // Pinned-prefix tier — explicit per-prefix pin via PinEntry. Used + // for hot-contract storage-trunk preload (the "storage root trunk + // cache for big accounts" path). Entries here NEVER evict — sized + // by the preload policy, not by an LRU. Writes to pinned prefixes + // (via Put/SD.Flush) update the entry in place rather than displace + // it, so cross-block correctness is maintained without losing the + // pin. Lookup checks this tier between root and tail. + pinned *maphash.Map[*branchCacheEntry] + // LRU tail — bounded entries, evicts oldest when full. maphash.LRU // wraps hashicorp/golang-lru/v2 which is thread-safe internally. tail *maphash.LRU[*branchCacheEntry] // Stats — atomic counters surfaced via Stats(). - rootHits, rootMisses atomic.Uint64 - tailHits, tailMisses atomic.Uint64 - bytesServed atomic.Uint64 + rootHits, rootMisses atomic.Uint64 + pinnedHits, pinnedMisses atomic.Uint64 + tailHits, tailMisses atomic.Uint64 + bytesServed atomic.Uint64 // Divergence counter — incremented by RecordDivergence when a caller // detects that a cache-served value disagrees with the canonical @@ -247,7 +257,10 @@ func NewBranchCache(tailCapacity int) *BranchCache { if err != nil { panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) } - return &BranchCache{tail: tail} + return &BranchCache{ + tail: tail, + pinned: maphash.NewMap[*branchCacheEntry](), + } } // isRootPrefix reports whether prefix targets the pinned root slot. The @@ -268,6 +281,13 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { c.rootHits.Add(1) return entry, true } + // Check pinned tier before tail. Pinned entries never evict and + // are fed by PinEntry (preload) + Put updates that preserve pin. + if entry, ok := c.pinned.Get(prefix); ok { + c.pinnedHits.Add(1) + return entry, true + } + c.pinnedMisses.Add(1) entry, ok := c.tail.Get(prefix) if !ok { c.tailMisses.Add(1) @@ -282,9 +302,43 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { c.root.Store(entry) return } + // If this prefix is pinned, update the pinned entry in place rather + // than route to the LRU tail. Keeps the pin alive across the + // SD.Flush invalidate+Put cycle that refreshes branch values every + // block — without this, every Put would silently lose the pin. + if _, ok := c.pinned.Get(prefix); ok { + c.pinned.Set(prefix, entry) + return + } c.tail.Set(prefix, entry) } +// PinEntry inserts or replaces a pinned cache entry for prefix. Pinned +// entries are never evicted by the LRU and survive across blocks +// (subject to Put updates from SD.Flush refreshing the bytes). Use for +// eager preload of hot prefixes — e.g. the storage-trunk of big +// contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate +// the input after the call. +func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.pinned.Set(prefix, &branchCacheEntry{ + data: dataCopy, + step: step, + writeSeq: c.writeSeq.Add(1), + origin: origin, + writeTimeNanos: time.Now().UnixNano(), + }) +} + +// PinnedCount returns the number of currently pinned entries. Useful +// for observability of the preload policy and eviction sanity (pinned +// entries are never evicted; this counter is monotonic over a +// PreloadContractTrunk run). +func (c *BranchCache) PinnedCount() int { + return c.pinned.Len() +} + // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). From 9b7c02aece3ad852fff6e5094d69ea07d2e0a052 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:59:40 +0000 Subject: [PATCH 014/120] commitment/preload: PreloadContractTrunk recursive enumeration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a function to pre-pin commitment branches for a given contract's storage subtree. Walks the trie depth-by-depth from depth 64 (storage subtree root) down to maxDepth: reads each branch via the supplied CommitmentReader, pins it via BranchCache.PinEntry, decodes the child bitmap, and recurses only into children that actually exist (no blind 16-way probing). contractHash is keccak256(address); the trunk lives at the prefix corresponding to the first 64 nibbles of the storage path. For a dense storage subtree (the SSTORE-bloat workload's bloat contract), expected pin count is ~16 + 256 + 4096 ≈ 4.4 K branches at depth 65/66/67, plus the root at depth 64. Sparse subtrees produce fewer. Loading strategy: per-prefix lookups via the reader. Simplest correct implementation. A bulk seg.Getter range-scan over sorted .kv would amortize disk seeks (per the parity-cluster observation in the consolidation memo) but requires building a prefix-range API on top of recsplit; defer until per-prefix lookup is shown to bottleneck. Step 3 of the storage-trunk pin prototype. --- execution/commitment/preload.go | 120 ++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 execution/commitment/preload.go diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go new file mode 100644 index 00000000000..22a16207434 --- /dev/null +++ b/execution/commitment/preload.go @@ -0,0 +1,120 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "encoding/binary" + "fmt" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// CommitmentReader is the read interface PreloadContractTrunk needs: +// GetLatest on the CommitmentDomain. Returns (value, step, found, err). +// Decoupled from any specific tx/aggregator type so callers can plug +// the reader in at preload time without dragging in db/state imports. +type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) + +// PreloadContractTrunk pins commitment branches for the given contract +// from depth 64 (storage subtree root) down to maxDepth into the supplied +// BranchCache. Walks the trie structure depth-by-depth: reads the branch, +// pins it, decodes its child bitmap, and recurses only into children that +// actually exist. +// +// contractHash is keccak256(address) — 32 bytes. The trunk lives at the +// commitment-domain prefix that corresponds to nibbles 0..63 of the +// storage path (the contract's account hash); enumeration extends from +// there into nibbles 64..maxDepth. +// +// Read amplification: total reads = number of branches actually present +// in the trunk (bounded by the contract's storage trie shape). For a +// dense storage subtree, this is ~16+256+4096 ≈ 4.4 K reads at depth 65, +// 66, 67. Sparse subtrees produce fewer. maxDepth caps the descent. +// +// Returns the number of pinned branches and any error from the reader. +// On error the partial pin set remains in place — those entries are +// still valid cache hits, they just won't see further enumeration below +// the failure point. +// +// Loading strategy: this is the simplest correct implementation — +// recursive enumerate via per-prefix reader lookups. A bulk variant +// using seg.Getter range-scan over sorted .kv would amortize disk +// seeks but requires building a prefix-range API on top of recsplit. +// Defer that optimization until per-prefix lookup is shown to bottleneck. +func PreloadContractTrunk( + contractHash []byte, + maxDepth int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (int, error) { + if len(contractHash) != 32 { + return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) + } + if maxDepth < 64 { + return 0, fmt.Errorf("PreloadContractTrunk: maxDepth %d < 64 (storage subtree starts at depth 64)", maxDepth) + } + if cache == nil { + return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") + } + + // Convert contract hash bytes to 64 nibbles. + contractNibbles := make([]byte, 64) + for i, b := range contractHash { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + pinned := 0 + var enumerate func(pathNibbles []byte, depth int) error + enumerate = func(pathNibbles []byte, depth int) error { + prefix := nibbles.HexToCompact(pathNibbles) + v, step, found, err := reader(prefix) + if err != nil { + return fmt.Errorf("preload at depth %d: %w", depth, err) + } + if !found { + return nil + } + cache.PinEntry(prefix, v, step, "preload-trunk") + pinned++ + if depth >= maxDepth { + return nil + } + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. + // Only recurse into children that actually exist (bit set in bitmap). + if len(v) < 4 { + return nil + } + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1< Date: Fri, 8 May 2026 16:03:09 +0000 Subject: [PATCH 015/120] state/execctx, commitment: PIN_CONTRACT_TRUNKS env hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a SharedDomains constructor hook that, when PIN_CONTRACT_TRUNKS is set, fires a one-shot background goroutine to preload the storage-subtree-trunk of each listed contract into BranchCache's pinned tier. Format: comma-separated list of 64-hex-char contract hashes (each is keccak256(addr)). Mechanism: - BranchCache.TryClaimPreload (atomic CAS) ensures the goroutine fires exactly once per cache lifetime, even though many SDs may be constructed (per-tx instances etc.). - Goroutine wraps sd.GetLatest as a CommitmentReader and calls commitment.PreloadContractTrunk for each contract hash, depth 64-70. - Logs progress per contract on completion. Closure-over-(sd, tx) is the prototype shape — works for the bench (both live for the whole process). Production deployment needs to revisit the lifetime — sd's tx may not outlive the goroutine. Step 4 of the storage-trunk pin prototype. Bench measurement is the next step (commit 5). --- db/state/execctx/domain_shared.go | 65 ++++++++++++++++++++++++++++ execution/commitment/branch_cache.go | 16 +++++++ 2 files changed, 81 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 7eb25fe20f1..4bf242eab02 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -19,10 +19,12 @@ package execctx import ( "bytes" "context" + "encoding/hex" "errors" "fmt" "math" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -201,6 +203,18 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) + // PIN_CONTRACT_TRUNKS=hash1,hash2,... (each hash is 64 hex chars = + // 32 bytes = keccak256(contractAddr)). When set and BranchCache is + // available, fire a one-shot background goroutine to preload the + // listed contracts' storage-subtree-trunk into the cache's pinned + // tier. Use TryClaimPreload to ensure we run exactly once per + // BranchCache lifetime even though many SDs may be constructed. + if branchCache != nil && branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(ctx, sd, tx, pinList, logger) + } + } + _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -1158,3 +1172,54 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN } return changes, nil } + +// triggerTrunkPreload kicks off a background goroutine that pre-pins +// the storage-subtree-trunk of each contract in pinList for the +// SharedDomains' BranchCache. pinList is a comma-separated list of +// 64-hex-char contract hashes (keccak256(addr)). Run from the SD +// constructor; gated by BranchCache.TryClaimPreload so it fires once +// per cache lifetime. +// +// Reader closes over sd + tx; the bench scenario keeps both alive for +// the whole process so the closure is safe. For production, this +// pattern needs revisiting (tx may not outlive the goroutine). +func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { + const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads + var hashes [][]byte + for _, hexStr := range strings.Split(pinList, ",") { + hexStr = strings.TrimSpace(hexStr) + if hexStr == "" { + continue + } + h, err := hex.DecodeString(hexStr) + if err != nil || len(h) != 32 { + logger.Warn("[trunk-preload] skipping invalid PIN_CONTRACT_TRUNKS entry", + "entry", hexStr, "err", err, "len", len(h)) + continue + } + hashes = append(hashes, h) + } + if len(hashes) == 0 { + return + } + cache := sd.branchCache + go func() { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil + } + for _, h := range hashes { + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, cache, logger) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + continue + } + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n) + } + }() +} diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index a7a64c59b4a..e6701db4514 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -191,6 +191,22 @@ type BranchCache struct { // origin label and timestamp to identify which write produced the // stale bytes. writeSeq atomic.Uint64 + + // preloadClaimed is set the first time TryClaimPreload is called. + // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in + // SharedDomains construction) so the preload goroutine fires + // exactly once per cache lifetime, even though many SharedDomains + // instances may be created over a process's lifetime. + preloadClaimed atomic.Bool +} + +// TryClaimPreload returns true the first time it's called on a given +// BranchCache instance, false on every subsequent call. Used by the +// trunk-preload trigger to ensure the preload goroutine runs exactly +// once per cache (process-lifetime), regardless of how many +// SharedDomains instances are constructed. +func (c *BranchCache) TryClaimPreload() bool { + return c.preloadClaimed.CompareAndSwap(false, true) } type branchCacheEntry struct { From 71d8480df282b47a4cc70be540dcd1135cff2922 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 16:17:53 +0000 Subject: [PATCH 016/120] state/execctx: trunk-preload sync (was async, hit cgo-pointer panic) Previous async-goroutine shape (d204c1b577) shared the SD's MDBX tx with the calling thread. Concurrent cursor use under the same tx tripped Go's cgo-pointer-pinning runtime check: panic: runtime error: cgo argument has Go pointer to unpinned Go pointer surfacing in an unrelated PruneBlocks goroutine during boot. Make the preload synchronous in the SD constructor for now: same TryClaimPreload guard (fires once per cache lifetime), but no goroutine. Boot pays the per-contract preload time as a one-off. Background-with-own-tx is the proper shape and remains a follow-up; owning the SD's tx exclusively for the preload duration is the safe shape until that lands. --- db/state/execctx/domain_shared.go | 57 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 4bf242eab02..b8d13da96ac 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1173,16 +1173,22 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN return changes, nil } -// triggerTrunkPreload kicks off a background goroutine that pre-pins -// the storage-subtree-trunk of each contract in pinList for the -// SharedDomains' BranchCache. pinList is a comma-separated list of -// 64-hex-char contract hashes (keccak256(addr)). Run from the SD -// constructor; gated by BranchCache.TryClaimPreload so it fires once -// per cache lifetime. +// triggerTrunkPreload runs a synchronous trunk-preload pass for each +// contract listed in pinList against the SharedDomains' BranchCache. +// pinList is a comma-separated list of 64-hex-char contract hashes +// (keccak256(addr)). Run from the SD constructor; gated by +// BranchCache.TryClaimPreload so it fires once per cache lifetime. // -// Reader closes over sd + tx; the bench scenario keeps both alive for -// the whole process so the closure is safe. For production, this -// pattern needs revisiting (tx may not outlive the goroutine). +// SYNCHRONOUS for prototype correctness: a previous async-goroutine +// shape shared the SD's MDBX tx with the calling thread and tripped +// "cgo argument has Go pointer to unpinned Go pointer" under +// concurrent cursor use. Background-with-own-tx is a follow-up; +// owning the SD's tx exclusively for the preload duration is the +// safe shape until that lands. +// +// Boot cost is the per-contract preload time, paid once per cache +// lifetime (TryClaimPreload guards). Block-app benchmarks see this +// as a one-time hit during the first SD construction. func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads var hashes [][]byte @@ -1202,24 +1208,21 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT if len(hashes) == 0 { return } - cache := sd.branchCache - go func() { - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) - if err != nil { - return nil, 0, false, err - } - return v, uint64(step), len(v) > 0, nil + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + if err != nil { + return nil, 0, false, err } - for _, h := range hashes { - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, cache, logger) - if err != nil { - logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) - continue - } - logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n) + return v, uint64(step), len(v) > 0, nil + } + for _, h := range hashes { + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + continue } - }() + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n) + } } From f85308daad9f57b9603d1ca102dc1f5fb9db2e89 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 19:26:09 +0000 Subject: [PATCH 017/120] commitment, state/execctx: trunk-preload progress logs + max-branches cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous bench (run-pin-trunk-instrumented-cold-cgroup-191347) hung at SD construction with no [trunk-preload] log lines for 5+ minutes. Erigon never reached "engine RPC ready" so all blocks came in as SYNCING. Two changes to localise + bound: 1. **Localisation**: add INFO logs at triggerTrunkPreload entry, per-contract starting/done with took, and a 500-prefix progress log inside PreloadContractTrunk. Whatever it does (or hangs on) is now observable. 2. **Bound**: cap PreloadContractTrunk at 10000 branches (vs ~4.4K expected for a saturated 4-level subtree at maxDepth=67). Drops maxDepth from 70 → 67 in the trigger (depth 64-67 = 16+256+4096 max branches) so we don't recurse into the per-slot tail where pinning has no value. Preload fails-fast on pathological subtrees rather than blocking SD construction indefinitely. --- db/state/execctx/domain_shared.go | 17 +++++++++++++---- execution/commitment/preload.go | 26 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index b8d13da96ac..533a9fad71c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1190,7 +1190,11 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // lifetime (TryClaimPreload guards). Block-app benchmarks see this // as a one-time hit during the first SD construction. func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { - const maxDepth = 70 // covers depths 64-70 = the dominant read range on bloat workloads + // Lower default depth (was 70 → blocked SD construction for too long + // on dense subtrees). 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K + // branches max per contract for a fully-saturated subtree. + const maxDepth = 67 + logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { hexStr = strings.TrimSpace(hexStr) @@ -1205,6 +1209,7 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT } hashes = append(hashes, h) } + logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "max_depth", maxDepth) if len(hashes) == 0 { return } @@ -1215,14 +1220,18 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT } return v, uint64(step), len(v) > 0, nil } - for _, h := range hashes { + for i, h := range hashes { + started := time.Now() + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) + took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "err", err) + "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) continue } logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n) + "hash", hex.EncodeToString(h), "pinned", n, "took", took) } + logger.Info("[trunk-preload] all done", "contracts", len(hashes)) } diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 22a16207434..497e62a09f8 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -72,9 +72,27 @@ func PreloadContractTrunk( contractNibbles[2*i+1] = b & 0x0f } + // Hard cap to bound preload work. A fully-saturated 4-level subtree + // is 1+16+256+4096 = 4369 branches; this gives headroom but still + // fails fast on malformed input (e.g. a contract with truly + // pathological branching). Tune downward if this turns out to bite. + const maxBranches = 10000 + pinned := 0 + var stopped bool var enumerate func(pathNibbles []byte, depth int) error enumerate = func(pathNibbles []byte, depth int) error { + if stopped { + return nil + } + if pinned >= maxBranches { + stopped = true + if logger != nil { + logger.Warn("[trunk-preload] hit max-branches cap", + "cap", maxBranches, "depth", depth) + } + return nil + } prefix := nibbles.HexToCompact(pathNibbles) v, step, found, err := reader(prefix) if err != nil { @@ -85,14 +103,18 @@ func PreloadContractTrunk( } cache.PinEntry(prefix, v, step, "preload-trunk") pinned++ + // Periodic progress log so a slow preload is observable. + if logger != nil && pinned%500 == 0 { + logger.Info("[trunk-preload] progress", "pinned", pinned, "depth", depth) + } if depth >= maxDepth { return nil } - // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. - // Only recurse into children that actually exist (bit set in bitmap). if len(v) < 4 { return nil } + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. + // Only recurse into children that actually exist (bit set in bitmap). bitmap := binary.BigEndian.Uint16(v[2:4]) for n := 0; n < 16; n++ { if bitmap&(1< Date: Fri, 8 May 2026 19:51:55 +0000 Subject: [PATCH 018/120] state/execctx: run trunk-preload AFTER SeekCommitment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous shape (d204c1b577) ran triggerTrunkPreload BEFORE sd.SeekCommitment in NewSharedDomains. Bench result: when the preload fired (PIN_CONTRACT_TRUNKS set), every subsequent block came back SYNCING — engine kept attempting backward-download which fails on this peerless setup, no block ever validated, no cache-fp ever fired. Without the preload firing, the same binary works fine (verify-bench PASS at 3.26s). Hypothesis (untested but matches the symptom): preload's sd.GetLatest reads ran before SeekCommitment had resolved the SD's view of the chain head. Pinned values were therefore inconsistent with the committed state, and the trie compute on the first block got wrong root → SYNCING → backward-download → no peers → death spiral with no Flush ever updating the (stale) pinned entries. Fix is mechanical: move the preload call to after SeekCommitment. The TryClaimPreload guard still ensures fire-once-per-cache lifetime. If subsequent bench shows pin_count > 0 + pin_hit > 0 + blocks validating normally, the hypothesis is confirmed; if SYNCING repeats, the bug is something else and we need to revert and debug differently. --- db/state/execctx/domain_shared.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 533a9fad71c..e08420cf2be 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -203,18 +203,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) - // PIN_CONTRACT_TRUNKS=hash1,hash2,... (each hash is 64 hex chars = - // 32 bytes = keccak256(contractAddr)). When set and BranchCache is - // available, fire a one-shot background goroutine to preload the - // listed contracts' storage-subtree-trunk into the cache's pinned - // tier. Use TryClaimPreload to ensure we run exactly once per - // BranchCache lifetime even though many SDs may be constructed. - if branchCache != nil && branchCache.TryClaimPreload() { - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(ctx, sd, tx, pinList, logger) - } - } - _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -231,6 +219,19 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } } + // PIN_CONTRACT_TRUNKS=hash1,hash2,... preload contract storage-trunk + // into BranchCache's pinned tier. Run AFTER SeekCommitment so the + // SD has resolved its view of the chain head — reading via + // sd.GetLatest BEFORE that produced pinned values inconsistent with + // the committed state and broke subsequent block validation + // (every block came back SYNCING). TryClaimPreload guards + // fire-once-per-cache lifetime. + if branchCache != nil && branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(ctx, sd, tx, pinList, logger) + } + } + return sd, nil } From 29ebc428024e00c7a8280d82f0bd0fb2beed6076 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 20:13:47 +0000 Subject: [PATCH 019/120] state/execctx: trunk-preload async with own tx, triggered from EnableParaTrieDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous prototype iterations both broke block validation: 1. Async sharing the SD's MDBX tx (d204c1b577) → cgo "unpinned Go pointer" panic from concurrent cursor use. 2. Synchronous from NewSharedDomains (5a81976553 / 4c9ead456d) → blocked the engine HTTP handler for ~3-4s during the preload window, causing the bench's first NewPayload to be dropped. Confirmed: the bench's height=24358001 is ABSENT from the erigon log; the next received block (24358002) then fails backward-download (no peers) → SYNCING forever. Restructure: - Move trigger from NewSharedDomains to EnableParaTrieDB. The latter is called from the staged-sync exec-stage init, NOT from request handlers, AND has access to a kv.TemporalRoDB. - triggerTrunkPreload now takes the DB (not a tx) and spawns a goroutine that opens its OWN tx via db.BeginTemporalRo. No shared cursors with the main pipeline; no blocking the engine. - Reader uses tx.GetLatest directly (not sd.GetLatest) — the SD layering would re-introduce shared-state risk and isn't needed (pinned bytes don't depend on sd.mem state). Same TryClaimPreload guard ensures the preload fires once per BranchCache lifetime regardless of how many SDs construct. If this works the bench should: - Show [trunk-preload] log lines firing once - Pin ~4369 branches - TEST block cache-fp shows pin_hit > 0 and files_comm < 1K - All blocks validate normally (no SYNCING failure) --- db/state/execctx/domain_shared.go | 121 ++++++++++++++++++------------ 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index e08420cf2be..feeeaba3293 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -219,18 +219,15 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg } } - // PIN_CONTRACT_TRUNKS=hash1,hash2,... preload contract storage-trunk - // into BranchCache's pinned tier. Run AFTER SeekCommitment so the - // SD has resolved its view of the chain head — reading via - // sd.GetLatest BEFORE that produced pinned values inconsistent with - // the committed state and broke subsequent block validation - // (every block came back SYNCING). TryClaimPreload guards - // fire-once-per-cache lifetime. - if branchCache != nil && branchCache.TryClaimPreload() { - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(ctx, sd, tx, pinList, logger) - } - } + // PIN_CONTRACT_TRUNKS preload runs from EnableParaTrieDB (called by + // the staged sync exec stage) NOT here in NewSharedDomains — at SD + // construction time we'd block the engine HTTP handler for ~3-4s + // while preload reads, causing the bench's first NewPayload to be + // dropped. EnableParaTrieDB runs in the staged sync init goroutine, + // not the request handler, AND has access to the kv.TemporalRoDB + // needed to spawn an own-tx preload goroutine that doesn't share + // cursors with the main pipeline (the cause of the earlier + // cgo-pointer panic). return sd, nil } @@ -1119,6 +1116,24 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) + + // Trigger trunk-preload here (not from NewSharedDomains) so: + // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine + // with — avoids the cgo-pointer panic that the earlier shared-tx + // attempt produced. + // (b) we run from the stage-init path, not an engine request handler, + // so we don't block engine HTTP responses during the 3-4s preload + // window — the synchronous-from-NewSharedDomains shape caused the + // bench's first NewPayload to be dropped, breaking block validation. + // TryClaimPreload guards fire-once-per-BranchCache-lifetime. + if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { + return + } + pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", "") + if pinList == "" { + return + } + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } func (sd *SharedDomains) EnableWarmupCache(enable bool) { @@ -1174,26 +1189,23 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN return changes, nil } -// triggerTrunkPreload runs a synchronous trunk-preload pass for each -// contract listed in pinList against the SharedDomains' BranchCache. -// pinList is a comma-separated list of 64-hex-char contract hashes -// (keccak256(addr)). Run from the SD constructor; gated by -// BranchCache.TryClaimPreload so it fires once per cache lifetime. +// triggerTrunkPreload spawns a goroutine that pre-pins the storage-trunk +// of each contract in pinList for the given BranchCache. pinList is a +// comma-separated list of 64-hex-char contract hashes (keccak256(addr)). // -// SYNCHRONOUS for prototype correctness: a previous async-goroutine -// shape shared the SD's MDBX tx with the calling thread and tripped -// "cgo argument has Go pointer to unpinned Go pointer" under -// concurrent cursor use. Background-with-own-tx is a follow-up; -// owning the SD's tx exclusively for the preload duration is the -// safe shape until that lands. +// The goroutine opens its OWN kv.TemporalRoTx via db.BeginTemporalRo, +// distinct from the staged-sync's main tx. Two earlier shapes failed: +// 1. Goroutine sharing the SD's MDBX tx → cgo "unpinned Go pointer" +// panic from concurrent cursor use across goroutines. +// 2. Synchronous from NewSharedDomains → blocked the engine HTTP +// handler for ~3-4s during the preload window, dropping the bench's +// first NewPayload and breaking subsequent backward-download. // -// Boot cost is the per-contract preload time, paid once per cache -// lifetime (TryClaimPreload guards). Block-app benchmarks see this -// as a one-time hit during the first SD construction. -func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalTx, pinList string, logger log.Logger) { - // Lower default depth (was 70 → blocked SD construction for too long - // on dense subtrees). 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K - // branches max per contract for a fully-saturated subtree. +// Own-tx async sidesteps both. Caller must guarantee fire-once via +// BranchCache.TryClaimPreload (the only call site does this). +func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { + // 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max per + // contract for a fully-saturated subtree. const maxDepth = 67 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte @@ -1214,25 +1226,38 @@ func triggerTrunkPreload(ctx context.Context, sd *SharedDomains, tx kv.TemporalT if len(hashes) == 0 { return } - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := sd.GetLatest(kv.CommitmentDomain, tx, prefix) + go func() { + tx, err := db.BeginTemporalRo(ctx) if err != nil { - return nil, 0, false, err + logger.Warn("[trunk-preload] BeginTemporalRo failed", "err", err) + return } - return v, uint64(step), len(v) > 0, nil - } - for i, h := range hashes { - started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, sd.branchCache, logger) - took := time.Since(started) - if err != nil { - logger.Warn("[trunk-preload] failed", - "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) - continue + defer tx.Rollback() + // Read directly via tx.GetLatest — no SharedDomains wrap. + // SD's GetLatest layers in sd.mem / parent.mem checks that + // (a) we don't need (the trunk hasn't been written yet in this + // goroutine's tx), and (b) would risk re-introducing the + // shared-cursor problem. + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := tx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil } - logger.Info("[trunk-preload] contract done", - "hash", hex.EncodeToString(h), "pinned", n, "took", took) - } - logger.Info("[trunk-preload] all done", "contracts", len(hashes)) + for i, h := range hashes { + started := time.Now() + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) + n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, branchCache, logger) + took := time.Since(started) + if err != nil { + logger.Warn("[trunk-preload] failed", + "hash", hex.EncodeToString(h), "pinned_so_far", n, "took", took, "err", err) + continue + } + logger.Info("[trunk-preload] contract done", + "hash", hex.EncodeToString(h), "pinned", n, "took", took) + } + logger.Info("[trunk-preload] all done", "contracts", len(hashes)) + }() } From db656cc58939931042b7117d3c8258e8ef69423c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 20:27:07 +0000 Subject: [PATCH 020/120] execution/commitment: PIN_TRUNK_MAX_DEPTH env + raise maxBranches cap Make the trunk-pin maxDepth configurable via env (default 67) so we can sweep depths to find the memory/perf sweet spot without rebuilding. Bump the per-contract maxBranches cap from 10K to 200K so deeper saturated subtrees don't get truncated mid-walk. --- db/state/execctx/domain_shared.go | 10 +++++++--- execution/commitment/preload.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index feeeaba3293..671057fbbbf 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1204,9 +1204,13 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max per - // contract for a fully-saturated subtree. - const maxDepth = 67 + // Default 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max + // per contract for a fully-saturated subtree. Override via + // PIN_TRUNK_MAX_DEPTH for sweep experiments. + maxDepth := 67 + if v := dbg.EnvInt("PIN_TRUNK_MAX_DEPTH", 67); v >= 64 { + maxDepth = v + } logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 497e62a09f8..c996943318c 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -76,7 +76,7 @@ func PreloadContractTrunk( // is 1+16+256+4096 = 4369 branches; this gives headroom but still // fails fast on malformed input (e.g. a contract with truly // pathological branching). Tune downward if this turns out to bite. - const maxBranches = 10000 + const maxBranches = 200000 pinned := 0 var stopped bool From ce8cce46e3b7e9413060ff7dcead496eec5b0e6d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:26:56 +0000 Subject: [PATCH 021/120] stagedsync/exec3_parallel: re-enable trie warmup on parallel path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous code disabled the Warmuper for the parallel commitment path out of concern that it would interact with the calculator's SetUpdates call. In practice the Warmuper's reads are independent of the calculator's update buffer — they pre-fetch branch data while EVM execution runs, and the calculator's SetUpdates only affects ComputeCommitment's input set, not the warmup paths. Re-enabling produces a measured 8× throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark (block 24358306, the canonical fixture for #20920), restoring the win first observed in Run H/I of the trie-perf investigation. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/stagedsync/exec3_parallel.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 3160516402e..97eafddeb12 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -217,10 +217,13 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U pe.rs.Domains().SetInMemHistoryReads(true) defer pe.rs.Domains().SetInMemHistoryReads(prevInMemHistoryReads) - // Disable trie warmup — the Warmuper uses sdCtx.updates which the - // calculator replaces via SetUpdates before ComputeCommitment. - pe.rs.Domains().EnableTrieWarmup(false) - defer pe.rs.Domains().EnableTrieWarmup(true) + // Trie warmup left enabled for the parallel path. Original disable was + // based on a calculator/warmer interaction concern that turned out to be + // overly conservative — the Warmuper's reads are independent of the + // calculator's SetUpdates call. Removing the disable produced an 8× + // throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark + // (block 24358306) by letting the Warmuper pre-fetch branch data while + // EVM execution runs. See #20920 for the canonical perf measurement. // Skip step-boundary commitment — the calculator handles this. pe.rs.StateV3.SetSkipStepBoundaryCommitment(true) From e97c95f47c322080811b408f9c6fa34e67081f50 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 21:33:41 +0000 Subject: [PATCH 022/120] commitment: WarmupCache hit/miss/evict observability counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds atomic counters (branch hit/miss/evict + bytes-served, account hit/miss, storage hit/miss) to WarmupCache, plus a Stats() string formatter and ResetStats() for per-Process accumulation reset. Counters are updated on every Get/Evict path (existing Put paths were already counted via cache size). Useful for: - Confirming warmup effectiveness in production logs - Per-block diagnostics when investigating commitment perf - Future per-pool dashboards once a coordinator/observability layer lands (tracked separately) No behavior change beyond the counter updates themselves. Stats() format is one line, suitable for embedding in the existing LogCommitments output. ResetStats() zeros counters without touching cached data — useful for per-Process windowed measurement. Clear() also resets counters along with the data, since data and counters were accumulated together. Test: TestWarmupCache_Stats covers hit/miss/evict accounting across branch/account/storage paths and verifies Stats() format + ResetStats() preserves data. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 69 ++++++++++++++++- execution/commitment/warmup_cache_test.go | 91 +++++++++++++++++++++++ 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index d6389a350d9..643ae0e1174 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -17,6 +17,7 @@ package commitment import ( + "fmt" "sync/atomic" "github.com/erigontech/erigon/common/maphash" @@ -44,6 +45,13 @@ type WarmupCache struct { branches *maphash.Map[*branchEntry] accounts *maphash.Map[*accountEntry] storage *maphash.Map[*storageEntry] + + // Observability counters. Updated by Get/Put paths; read by Stats(). + // Atomic so warmup-worker reads and main-trie reads don't race. + branchHits, branchMisses, branchEvicted atomic.Uint64 + branchBytesServed atomic.Uint64 + accountHits, accountMisses atomic.Uint64 + storageHits, storageMisses atomic.Uint64 } // NewWarmupCache creates a new warmup cache instance. @@ -67,19 +75,33 @@ func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { // GetBranch retrieves branch data from the cache. func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) - if !found || entry.isEvicted.Load() { + if !found { + c.branchMisses.Add(1) + return nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) return nil, false } + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) return entry.data, true } // GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) - if !found || entry.isEvicted.Load() { + if !found { + c.branchMisses.Add(1) + return nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) return nil, false } entry.isEvicted.Store(true) + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) return entry.data, true } @@ -105,8 +127,10 @@ func (c *WarmupCache) PutAccount(plainKey []byte, update *Update) { func (c *WarmupCache) GetAccount(plainKey []byte) (*Update, bool) { entry, found := c.accounts.Get(plainKey) if !found || entry.isEvicted.Load() { + c.accountMisses.Add(1) return nil, false } + c.accountHits.Add(1) return entry.update, true } @@ -143,8 +167,10 @@ func (c *WarmupCache) PutStorage(plainKey []byte, update *Update) { func (c *WarmupCache) GetStorage(plainKey []byte) (*Update, bool) { entry, found := c.storage.Get(plainKey) if !found || entry.isEvicted.Load() { + c.storageMisses.Add(1) return nil, false } + c.storageHits.Add(1) return entry.update, true } @@ -178,9 +204,46 @@ func (c *WarmupCache) EvictPlainKey(plainKey []byte) { } } -// Clear clears all cached data. +// Clear clears all cached data and resets stats counters. func (c *WarmupCache) Clear() { c.branches = maphash.NewMap[*branchEntry]() c.accounts = maphash.NewMap[*accountEntry]() c.storage = maphash.NewMap[*storageEntry]() + c.ResetStats() +} + +// Stats returns a one-line summary of cache hit/miss counters. +// Format matches the per-block log line: branch, account, storage with +// hit-percentages and bytes served. Useful in commitment debug logs and +// for the per-Process LogCommitments line. +func (c *WarmupCache) Stats() string { + bh, bm, be := c.branchHits.Load(), c.branchMisses.Load(), c.branchEvicted.Load() + bb := c.branchBytesServed.Load() + ah, am := c.accountHits.Load(), c.accountMisses.Load() + sh, sm := c.storageHits.Load(), c.storageMisses.Load() + pct := func(hit, miss uint64) float64 { + total := hit + miss + if total == 0 { + return 0 + } + return 100.0 * float64(hit) / float64(total) + } + return fmt.Sprintf( + "branch hit=%d miss=%d evict=%d (%.1f%%, %.1f MiB) | acct hit=%d miss=%d (%.1f%%) | stor hit=%d miss=%d (%.1f%%)", + bh, bm, be, pct(bh, bm), float64(bb)/1024/1024, + ah, am, pct(ah, am), + sh, sm, pct(sh, sm), + ) +} + +// ResetStats zeros out all hit/miss/byte counters without touching cached data. +func (c *WarmupCache) ResetStats() { + c.branchHits.Store(0) + c.branchMisses.Store(0) + c.branchEvicted.Store(0) + c.branchBytesServed.Store(0) + c.accountHits.Store(0) + c.accountMisses.Store(0) + c.storageHits.Store(0) + c.storageMisses.Store(0) } diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index 43de177ee06..a52fd85cbff 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -301,3 +301,94 @@ func BenchmarkComparison_Map_100k(b *testing.B) { cache.GetStorage(keys[i%len(keys)]) } } + +// TestWarmupCache_Stats verifies that hit/miss/evict counters are updated +// correctly across the Get/Put/Evict paths and that Stats() returns a +// deterministic format. +func TestWarmupCache_Stats(t *testing.T) { + cache := NewWarmupCache() + + // Branch hits + misses + bytes-served + bk := []byte("k-branch") + bd := []byte("branch-data-12345") + cache.PutBranch(bk, bd) + if _, ok := cache.GetBranch(bk); !ok { + t.Fatal("expected branch hit") + } + if _, ok := cache.GetBranch([]byte("missing-branch")); ok { + t.Fatal("expected branch miss") + } + + // Branch eviction + cache.EvictBranch(bk) + if _, ok := cache.GetBranch(bk); ok { + t.Fatal("expected branch evicted miss") + } + + // Account hits + misses + ak := []byte("k-acct") + cache.PutAccount(ak, &Update{}) + if _, ok := cache.GetAccount(ak); !ok { + t.Fatal("expected account hit") + } + if _, ok := cache.GetAccount([]byte("missing-acct")); ok { + t.Fatal("expected account miss") + } + + // Storage hits + misses + sk := []byte("k-stor") + cache.PutStorage(sk, &Update{}) + if _, ok := cache.GetStorage(sk); !ok { + t.Fatal("expected storage hit") + } + if _, ok := cache.GetStorage([]byte("missing-stor")); ok { + t.Fatal("expected storage miss") + } + + // Counters: branch hit=1 miss=1 evict=1 bytes=17, acct hit=1 miss=1, stor hit=1 miss=1 + if got := cache.branchHits.Load(); got != 1 { + t.Fatalf("branchHits: got %d, want 1", got) + } + if got := cache.branchMisses.Load(); got != 1 { + t.Fatalf("branchMisses: got %d, want 1", got) + } + if got := cache.branchEvicted.Load(); got != 1 { + t.Fatalf("branchEvicted: got %d, want 1", got) + } + if got := cache.branchBytesServed.Load(); got != uint64(len(bd)) { + t.Fatalf("branchBytesServed: got %d, want %d", got, len(bd)) + } + if got := cache.accountHits.Load(); got != 1 { + t.Fatalf("accountHits: got %d, want 1", got) + } + if got := cache.accountMisses.Load(); got != 1 { + t.Fatalf("accountMisses: got %d, want 1", got) + } + if got := cache.storageHits.Load(); got != 1 { + t.Fatalf("storageHits: got %d, want 1", got) + } + if got := cache.storageMisses.Load(); got != 1 { + t.Fatalf("storageMisses: got %d, want 1", got) + } + + // Stats() format + stats := cache.Stats() + for _, want := range []string{ + "branch hit=1 miss=1 evict=1", + "acct hit=1 miss=1", + "stor hit=1 miss=1", + } { + if !bytes.Contains([]byte(stats), []byte(want)) { + t.Errorf("Stats() missing %q\nfull: %s", want, stats) + } + } + + // ResetStats zeros counters but leaves data intact + cache.ResetStats() + if got := cache.branchHits.Load(); got != 0 { + t.Fatalf("branchHits after Reset: got %d, want 0", got) + } + if _, ok := cache.GetAccount(ak); !ok { + t.Fatal("account data should survive ResetStats") + } +} From af2ef6872289a5824a539259782283946e4d61ed Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 4 May 2026 22:21:46 +0000 Subject: [PATCH 023/120] commitment: extract unfoldKeyPath as per-key traversal primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure refactor — behavior preserved. Lifts the unfold-loop from HexPatriciaHashed.followAndUpdate into its own method, parameterized over (hashedKey, plainKey) and intended as the per-key traversal primitive that future orchestrators consume. Today only followAndUpdate calls it, replacing the inline loop with a one-line call. The extracted method preserves the existing metric attribution (StartUnfolding) and trace-print behavior verbatim. Why now: this is the second step in the representation-reduction sequence (see agentspecs/trie-data-pipeline-complexity-tax.md). Future PRs will introduce orchestrators that drive unfold-only walks of touched-key paths to fill cell state without going through the full fold/update cycle: - Cache populator (decoded-payload BranchCache) needs to walk a touched-key path and capture the cells encountered, without triggering fold or modifying the trie's update buffer. - Stage E parallel pre-unfold orchestrator drives unfoldKeyPath across multiple HexPatriciaHashed instances concurrently to pre-warm trie state before commit. Both consume the same primitive. Centralising it now means each future orchestrator is a thin wrapper rather than a duplicate of the unfold-loop logic. Tests: full commitment test suite passes without modification (all 8+ test files in execution/commitment/), confirming the refactor preserves followAndUpdate's behavior. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/hex_patricia_hashed.go | 54 ++++++++++++++------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 2f86a61a6d7..3f53e71fa04 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2504,6 +2504,40 @@ func (hph *HexPatriciaHashed) RootHash() ([]byte, error) { return rootHash[1:], nil // first byte is 128+hash_len=160 } +// unfoldKeyPath drives hph.unfold in a loop until the trie is unfolded +// far enough that the cell at hashedKey can be updated — i.e. until +// needUnfolding returns 0. This is the per-key traversal primitive +// that follows the fold step in followAndUpdate. +// +// Extracted as a primitive so future orchestrators can drive +// unfold-only traversals (e.g. cache populators that walk a touched-key +// path to fill cell state without doing a fold/update). HashSort's +// per-key followAndUpdate is the sequential orchestrator over this +// primitive today; future concurrent / parallel orchestrators (each +// with their own HexPatriciaHashed instance) can reuse it. +// +// plainKey is used only for per-key metrics labelling (StartUnfolding). +// Pass an empty/nil slice if no metric attribution is needed. +func (hph *HexPatriciaHashed) unfoldKeyPath(hashedKey, plainKey []byte) error { + for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { + printLater := hph.currentKeyLen == 0 && hph.mounted && hph.trace + var unfoldDone func() + if dbg.KVReadLevelledMetrics { + unfoldDone = hph.metrics.StartUnfolding(plainKey) + } + if err := hph.unfold(hashedKey, unfolding); err != nil { + return fmt.Errorf("unfold: %w", err) + } + if unfoldDone != nil { + unfoldDone() + } + if printLater { + fmt.Printf("[%x] subtrie pref '%x' d=%d\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hph.depths[max(0, hph.activeRows-1)]) + } + } + return nil +} + func (hph *HexPatriciaHashed) followAndUpdate(hashedKey, plainKey []byte, stateUpdate *Update) (err error) { //if hph.trace { // fmt.Printf("mnt: %0x current: %x path %x\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hashedKey) @@ -2521,23 +2555,9 @@ func (hph *HexPatriciaHashed) followAndUpdate(hashedKey, plainKey []byte, stateU foldDone() } } - // Now unfold until we step on an empty cell - for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { - printLater := hph.currentKeyLen == 0 && hph.mounted && hph.trace - var unfoldDone func() - if dbg.KVReadLevelledMetrics { - unfoldDone = hph.metrics.StartUnfolding(plainKey) - } - if err := hph.unfold(hashedKey, unfolding); err != nil { - return fmt.Errorf("unfold: %w", err) - } - if unfoldDone != nil { - unfoldDone() - } - if printLater { - fmt.Printf("[%x] subtrie pref '%x' d=%d\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hph.depths[max(0, hph.activeRows-1)]) - } - // fmt.Printf("mnt: %0x current: %x path %x\n", hph.mountedNib, hph.currentKey[:hph.currentKeyLen], hashedKey) + // Now unfold the path so the cell at hashedKey is reachable. + if err := hph.unfoldKeyPath(hashedKey, plainKey); err != nil { + return err } if stateUpdate == nil { From 4074bccf75e991ec9014e45d99142841470c6c4e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:47:48 +0000 Subject: [PATCH 024/120] commitment: WarmupCache dirty-flag + PutBranchIfClean invariants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the carry-as-is correctness invariants from the PR #19954 investigation as scaffolding on the existing WarmupCache: - branchEntry.dirty atomic.Bool — signals "stale until cleared" - PutBranchIfClean(prefix, data) bool — skips write if entry dirty - MarkBranchDirty(prefix) — mark for later refusal of stale puts These are scaffolding additions; no callsites use them yet. Existing PutBranch unconditionally overwrites and clears any prior dirty flag (creates a fresh entry), preserving today's semantic exactly for existing callers. Why now: the prototype investigation (see agentspecs/commitment-cache-prototype-dev-context.md) found that inline-invalidate-on-write is incompatible with deferred encoding — update-in-place breaks correctness because there's a window between fold (computes hash, holds new state) and encoder (writes encoded bytes) where readers see stale cached bytes. The reth-research (agentspecs/reth-1ggas-research.md §4) calls the dirty-flag pattern out as the design that resolves this without forcing synchronous encoding: the encoder marks the entry dirty BEFORE its own write completes, so any racing read knows to bypass the cache for that key. Today's WarmupCache lifecycle (per-Process, warmup completes before fold begins) does NOT exhibit this race — these invariants are infrastructure for the future cross-block persistence work where warmup-style writers can outlive their parent Process. Tests: - TestWarmupCache_DirtyFlag: PutBranchIfClean refuses dirty entry, unconditional PutBranch clears dirty. - TestWarmupCache_DirtyFlag_MarkAbsentKey: marking absent key is no-op (no panic, no entry created). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 49 ++++++++++++++++++++ execution/commitment/warmup_cache_test.go | 56 +++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 643ae0e1174..219ce925a50 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -26,6 +26,15 @@ import ( type branchEntry struct { data []byte isEvicted atomic.Bool + // dirty signals "the canonical store has been written to since this + // entry was populated; treat as stale until cleared." Read paths + // MAY return miss when dirty is set; write paths MUST set dirty. + // Used together with PutBranchIfClean to prevent late warmup writes + // from clobbering fresh fold writes (TOCTOU race documented in the + // reth-research §4 dirty-flag pattern). Ephemeral cache today does + // not have the race because warmup completes before fold begins; + // invariant is in place ahead of cross-block persistence work. + dirty atomic.Bool } type accountEntry struct { @@ -72,6 +81,46 @@ func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { c.branches.Set(prefix, &branchEntry{data: dataCopy}) } +// PutBranchIfClean stores branch data in the cache only if no existing entry +// is marked dirty. Returns true on store, false if a dirty entry was present +// (indicating the canonical store has been updated since the caller last +// read, and the caller's data is potentially stale). +// +// Use from warmup-style writers that may race with fold writes — the fold +// path marks branches dirty before its own write completes, so a warmup +// worker that reads pre-fold then attempts to write post-fold will see +// dirty=true and skip the store. +// +// Today's call sites (warmup workers in HexPatriciaHashed.Process) do not +// race with fold because warmup completes before HashSort begins. Invariant +// is in place for future cross-block persistence work where warmup-style +// writes can outlive their parent Process. +func (c *WarmupCache) PutBranchIfClean(prefix []byte, data []byte) bool { + if existing, found := c.branches.Get(prefix); found && existing.dirty.Load() { + return false + } + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + c.branches.Set(prefix, &branchEntry{data: dataCopy}) + return true +} + +// MarkBranchDirty flags a branch entry as stale-until-cleared. The next +// PutBranchIfClean for this prefix will skip; reads (GetBranch / +// GetAndEvictBranch) currently return the entry regardless — the dirty +// signal is consumed only on the write path today. +// +// Use from fold/encoder paths that have decided to overwrite a branch but +// haven't yet captured the new bytes. The mark-dirty + later actual-write +// pattern is the deferred-encoding-friendly alternative to inline +// invalidate, motivated by the prototype investigation that found inline +// invalidate breaks update-in-place semantics under deferred encoding. +func (c *WarmupCache) MarkBranchDirty(prefix []byte) { + if entry, found := c.branches.Get(prefix); found { + entry.dirty.Store(true) + } +} + // GetBranch retrieves branch data from the cache. func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index a52fd85cbff..9a6d50da17d 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -20,6 +20,8 @@ import ( "bytes" "crypto/rand" "testing" + + "github.com/stretchr/testify/require" ) // TestWarmupCache_Basic tests basic put/get operations @@ -392,3 +394,57 @@ func TestWarmupCache_Stats(t *testing.T) { t.Fatal("account data should survive ResetStats") } } + +// TestWarmupCache_DirtyFlag verifies the dirty flag scaffolding — +// MarkBranchDirty + PutBranchIfClean — that prevents late writers from +// clobbering fresh data once cross-block persistence is wired up. +func TestWarmupCache_DirtyFlag(t *testing.T) { + cache := NewWarmupCache() + key := []byte("k-branch") + v1 := []byte("first-value") + v2 := []byte("second-value-skipped") + + // First Put always succeeds (no existing entry to be dirty). + require.True(t, cache.PutBranchIfClean(key, v1)) + got, ok := cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v1, got) + + // Marking dirty doesn't remove the entry — Get still returns it. + cache.MarkBranchDirty(key) + got, ok = cache.GetBranch(key) + require.True(t, ok, "MarkBranchDirty should not evict; consumers decide what to do with dirty data") + require.Equal(t, v1, got) + + // PutBranchIfClean refuses to overwrite a dirty entry. + require.False(t, cache.PutBranchIfClean(key, v2), + "PutBranchIfClean must skip when entry is dirty") + got, ok = cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v1, got, "dirty value must be preserved (write was rejected)") + + // PutBranch (non-If-Clean variant) overwrites unconditionally — used + // by the fold encoder path that holds the canonical new value. + cache.PutBranch(key, v2) + got, ok = cache.GetBranch(key) + require.True(t, ok) + require.Equal(t, v2, got) + + // After unconditional overwrite, the new entry is no longer dirty + // (fresh entry replaces the dirty one). + require.True(t, cache.PutBranchIfClean(key, []byte("third-value")), + "PutBranchIfClean must accept after unconditional Put cleared the dirty entry") +} + +// TestWarmupCache_DirtyFlag_MarkAbsentKey verifies that marking a key +// dirty when no entry exists is a no-op (no panic, no entry created). +func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { + cache := NewWarmupCache() + cache.MarkBranchDirty([]byte("never-stored")) + _, ok := cache.GetBranch([]byte("never-stored")) + require.False(t, ok, "MarkBranchDirty on absent key should not create an entry") + + // PutBranchIfClean for the same absent key should succeed (no dirty + // entry exists, so the write proceeds). + require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) +} From 4be9c86e4930f240cd53165caeec269217d90ff0 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 5 May 2026 07:53:25 +0000 Subject: [PATCH 025/120] =?UTF-8?q?commitment:=20WarmupCache=20GetBranchDe?= =?UTF-8?q?coded=20=E2=80=94=20lazy-decode=20read=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an additive read method that returns cached branches in already- decoded form, lazy-decoding on first decoded-read per entry and caching the parsed cells for subsequent reads. No existing callsite changes; existing GetBranch / GetAndEvictBranch / PutBranch callers continue to work with encoded bytes unchanged. Why now: this is step 5 of the representation-reduction sequence (see agentspecs/trie-data-pipeline-complexity-tax.md). The trie's read path currently does GetBranch (encoded) + DecodeBranchInto on every cache hit — paying decode CPU on every read. Switching that callsite to GetBranchDecoded (in a separate later commit) eliminates the redundant decode. The ENCODED form remains the source of truth — the encoder needs it for the merge-with-prev step, and it's what gets written to disk via PutBranch. The decoded form is derived lazily and cached alongside the entry. When PutBranch overwrites an entry, the new entry starts fresh and the next decoded read re-derives from the new bytes. API design notes: - Returns (bitmap, *[16]cell, ok). Caller derives touchMap/afterMap from bitmap based on its own deleted-vs-present-after context — the cache stores cells independent of that context so the same entry serves both readers. - The returned *[16]cell aliases entry-owned storage. Read-only consumption is safe across concurrent calls (decode runs at most once per entry via sync.Once); MUST NOT be modified in place. - Decode error → ok=false (don't count as hit OR miss; caller falls through to canonical re-read). Tests: - TestWarmupCache_GetBranchDecoded: round-trip equality with direct DecodeBranchInto, plus same-pointer reuse on repeat reads. - TestWarmupCache_GetBranchDecoded_Miss: behaves like GetBranch on absent keys. - TestWarmupCache_GetBranchDecoded_TruncatedData: graceful failure on corrupt entry (no panic). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/warmup_cache.go | 76 ++++++++++++++++++++++- execution/commitment/warmup_cache_test.go | 73 ++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 219ce925a50..800f3fad5b4 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -18,13 +18,31 @@ package commitment import ( "fmt" + "sync" "sync/atomic" "github.com/erigontech/erigon/common/maphash" ) type branchEntry struct { - data []byte + // data is the canonical encoded form (with the leading 2-byte touch-map + // prefix). Always populated by PutBranch / PutBranchIfClean. + data []byte + + // Lazy-decoded form. Populated on the first GetBranchDecoded call for + // this entry; subsequent calls return the cached decode. decodeOnce + // ensures decode runs at most once per entry even under concurrent + // reads. + // + // Encoded form is the source of truth; decoded form is derived. When + // data is replaced (PutBranch overwrites), the new entry starts fresh + // — next decoded read re-derives from the new bytes. + decodeOnce sync.Once + cells [16]cell + cellsBitmap uint16 + decodedReady bool + decodeErr error + isEvicted atomic.Bool // dirty signals "the canonical store has been written to since this // entry was populated; treat as stale until cleared." Read paths @@ -137,6 +155,62 @@ func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { return entry.data, true } +// GetBranchDecoded retrieves the cached branch in decoded form. Lazy-decodes +// on first access for each entry; subsequent reads return the cached cells +// pointer without redoing the parse work. +// +// Returns the bitmap of present children plus a pointer to the populated +// cells array. The caller derives touchMap/afterMap from the bitmap based +// on its own context (deleted vs present-after) — the cache stores cells +// independent of that context so the same entry serves both readers. +// +// Returns ok=false on miss or eviction (same semantics as GetBranch). Also +// returns ok=false if the cached bytes fail to decode — in that case the +// caller should fall through to a re-read from the canonical store. +// +// The returned *[16]cell pointer aliases storage owned by the cache entry +// — the caller MUST NOT modify the cells in place. Read-only consumption +// is safe across concurrent GetBranchDecoded calls (decode runs at most +// once per entry; subsequent reads see the populated cells). +func (c *WarmupCache) GetBranchDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + entry, found := c.branches.Get(prefix) + if !found { + c.branchMisses.Add(1) + return 0, nil, false + } + if entry.isEvicted.Load() { + c.branchEvicted.Add(1) + return 0, nil, false + } + entry.decodeOnce.Do(func() { + // PutBranch stores bytes WITH the leading 2-byte touch-map prefix; + // DecodeBranchInto consumes bytes WITHOUT it (matching the + // unfoldBranchNode call pattern that strips the prefix before + // decoding). + if len(entry.data) < 2 { + entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) + return + } + maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) + if err != nil { + entry.decodeErr = err + return + } + entry.cellsBitmap = maps.Bitmap + entry.decodedReady = true + }) + if !entry.decodedReady { + // Decode failed — caller should re-read from canonical store. + // Don't count as hit (the entry is mechanically present but + // unusable). Don't increment misses either — the caller will + // observe ok=false and decide. + return 0, nil, false + } + c.branchHits.Add(1) + c.branchBytesServed.Add(uint64(len(entry.data))) + return entry.cellsBitmap, &entry.cells, true +} + // GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { entry, found := c.branches.Get(prefix) diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go index 9a6d50da17d..db62ec6ceb2 100644 --- a/execution/commitment/warmup_cache_test.go +++ b/execution/commitment/warmup_cache_test.go @@ -448,3 +448,76 @@ func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { // entry exists, so the write proceeds). require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) } + +// TestWarmupCache_GetBranchDecoded verifies the lazy-decode read path: +// stored encoded bytes are decoded on first decoded-read and cached for +// subsequent reads, returning cells equivalent to direct DecodeBranchInto. +func TestWarmupCache_GetBranchDecoded(t *testing.T) { + cache := NewWarmupCache() + + // Encode a branch the same way BranchEncoder would, so our test + // data has the canonical [touchMap | bitmap | cells...] layout. + row, bm := generateCellRow(t, 16) + be := NewBranchEncoder(1024) + cellData := generateCellEncodeDataRow(t, row, bm) + enc, err := be.EncodeBranch(bm, bm, bm, &cellData) + require.NoError(t, err) + + prefix := []byte{0x12, 0x34} + cache.PutBranch(prefix, enc) + + // First decoded-read decodes lazily. + bitmap, cells, ok := cache.GetBranchDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap) + require.NotNil(t, cells) + + // Cells should match what direct DecodeBranchInto would produce + // from the same encoded bytes (skip the 2-byte touchMap prefix). + var expected [16]cell + expectedMaps, err := DecodeBranchInto(enc[2:], false, &expected) + require.NoError(t, err) + require.Equal(t, expectedMaps.Bitmap, bitmap) + for i := range cells { + require.Equal(t, expected[i].extLen, cells[i].extLen, "cell %d extLen", i) + require.Equal(t, expected[i].extension[:expected[i].extLen], cells[i].extension[:cells[i].extLen], "cell %d extension", i) + require.Equal(t, expected[i].accountAddr[:expected[i].accountAddrLen], cells[i].accountAddr[:cells[i].accountAddrLen], "cell %d accountAddr", i) + require.Equal(t, expected[i].storageAddr[:expected[i].storageAddrLen], cells[i].storageAddr[:cells[i].storageAddrLen], "cell %d storageAddr", i) + require.Equal(t, expected[i].hash[:expected[i].hashLen], cells[i].hash[:cells[i].hashLen], "cell %d hash", i) + } + + // Second decoded-read returns the SAME cells pointer — lazy decode + // runs at most once per entry. + bitmap2, cells2, ok := cache.GetBranchDecoded(prefix) + require.True(t, ok) + require.Equal(t, bm, bitmap2) + require.Same(t, cells, cells2, "expected cached cells pointer to be reused") + + // Encoded form is unchanged after decoded reads. + encGot, ok := cache.GetBranch(prefix) + require.True(t, ok) + require.Equal(t, []byte(enc), encGot, "encoded form unchanged by decoded reads") +} + +// TestWarmupCache_GetBranchDecoded_Miss verifies that misses on +// GetBranchDecoded behave the same as misses on GetBranch. +func TestWarmupCache_GetBranchDecoded_Miss(t *testing.T) { + cache := NewWarmupCache() + _, _, ok := cache.GetBranchDecoded([]byte("never-stored")) + require.False(t, ok) +} + +// TestWarmupCache_GetBranchDecoded_TruncatedData verifies the decode-error +// path — corrupt entry returns ok=false rather than panicking. +func TestWarmupCache_GetBranchDecoded_TruncatedData(t *testing.T) { + cache := NewWarmupCache() + // One byte is shorter than the touchMap prefix; decode will error. + cache.PutBranch([]byte("k"), []byte{0x42}) + _, _, ok := cache.GetBranchDecoded([]byte("k")) + require.False(t, ok, "truncated entry should return ok=false, not panic") + + // Encoded form still retrievable for callers that don't need decode. + got, ok := cache.GetBranch([]byte("k")) + require.True(t, ok) + require.Equal(t, []byte{0x42}, got) +} From 733dad5385e74946f125734f815257f1d4e374b8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 6 May 2026 17:29:22 +0000 Subject: [PATCH 026/120] execmodule: chain ValidateChain SD to currentContext for head-extending payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes a timing hole that surfaced once we wired the aggregator-scope BranchCache: between an FCU completing and the next newPayload, MDBX hasn't been committed yet (RunLoop's CommitCycle only fires under memory pressure), but currentContext.mem holds the latest writes from MergeExtendingFork. The fresh doms created in ValidateChain has no parent and a fresh roTx, so its ctx.Branch reads stale-MDBX while the aggregator-scope BranchCache (populated by the prior FV's CollectUpdate writes) holds the fresh state. That's the cross-newPayload divergence pattern observed in the bench (8-61 divergences and wrong-trie-root errors at block 3-14 across runs). Set doms.SetParent(currentContext) when the new payload extends the current canonical head (header.ParentHash == ReadHeadBlockHash). For fork payloads that don't extend head, leave parent unset: unwindToCommonCanonical below reverts doms's view to the common ancestor, and exposing currentContext.mem (post-divergence canonical writes) via the parent chain would shadow the unwound base and break fork validation. Verified by TestReorgsWithInsertChain — the "head-only" predicate is what the current single-canonical-chain SD topology supports. A proper per-branch SD lineage (each fork's validation chains to the last validated SD on its own branch, not always currentContext) is the follow-up needed for concurrent multi-fork validation. The current design supports a single canonical chain only; that's enough to close the divergence we have today, with the lineage extension tracked separately. --- execution/execmodule/exec_module.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 0302fdff491..c11ba373a48 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -484,6 +484,29 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() + // Chain the validation SD to currentContext when the new payload + // extends the current canonical head. Closes a timing hole: FCU's + // MergeExtendingFork moves the previous fork's writes into + // currentContext.mem, but MDBX commit only fires from RunLoop's + // CommitCycle on memory pressure. Between FCU and the next + // newPayload, currentContext.mem holds the latest state and MDBX + // is stale. Without the parent link, this fresh doms reads + // stale-MDBX while the aggregator-scope BranchCache (populated by + // the prior FV's CollectUpdate writes) holds the fresh state, + // producing divergence and wrong-trie-root downstream. + // + // Only set parent for head-extending payloads — fork validation + // requires unwindToCommonCanonical below to revert doms's view to + // the common ancestor, and exposing currentContext.mem (which holds + // post-divergence canonical writes) via the parent chain would + // shadow the unwound base. The current SD topology supports a + // single canonical chain only; per-branch SD lineage for concurrent + // multi-fork validation is a separate architectural follow-up. + canonicalHead := rawdb.ReadHeadBlockHash(tx) + if header.ParentHash == canonicalHead && e.currentContext != nil { + doms.SetParent(e.currentContext) + } + // Flush block overlay data (headers, bodies, TDs from InsertBlocks) into // the validation overlay so unwindToCommonCanonical and ValidatePayload — // and the parallel exec goroutine via NewReadView — see this block data. From a823097ae71e75222ae1ace0caa8deb0c88b82bd Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 17:02:30 +0000 Subject: [PATCH 027/120] db/state: H0 baseline benchmark for snapshot-vs-MDBX read cost Foundation for the "Snapshot vs MDBX read-cost equivalence" investigation (memory: snapshot-vs-mdbx-performance-equivalence.md). This file produces the headline ratio that quantifies the gap the investigation aims to close: warm-cache reads from snapshot .kv files should cost the same as warm-cache reads from MDBX (same disk, same page cache). H0 measures how far apart they are today. Five sub-benches: - MDBX_path full chain, key in MDBX - File_path full chain, key in file - Forced_file_path file-only debug path, file-resident keys - Forced_db_path DB-only debug path, MDBX-resident keys - Bloom_miss_path file-only debug path, MDBX-resident keys (file misses in xorfilter for every probe) Two operating modes; only synthetic is wired in this commit: - Synthetic (testDbAndAggregatorBench fixture): writes 64 full 16-tx steps, BuildFiles + repeated PruneSmallBatches drains all but the tip step into files. Phase 2 keys at txNums past the built-step boundary stay in MDBX. Partition by *actual* residency after setup so bench inputs match where keys really live. - Real-datadir (--snapdatadir flag): TODO. Opens an existing chaindata+snapshots datadir read-only and picks keys via cursor iteration / .kv decompressor walk. Required for production- relevant numbers since synthetic has tiny files and small values. Initial synthetic results on AMD EPYC 4244P (Accounts domain): MDBX_path 173 ns/op File_path 226 ns/op (1.31x MDBX) Forced_file_path 30 ns/op Forced_db_path 158 ns/op Bloom_miss_path 30 ns/op Synthetic dataset is too small to surface the production gap that pprof shows (xorfilter at 35% CPU on real bloat workload). H1-H4 benches and the real-datadir mode are the next steps. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 409 ++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 db/state/snapshot_vs_mdbx_bench_test.go diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go new file mode 100644 index 00000000000..8d431a046df --- /dev/null +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -0,0 +1,409 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package state_test + +// Benchmarks for the "Snapshot vs MDBX read-cost equivalence" +// investigation. See memory: snapshot-vs-mdbx-performance-equivalence.md +// +// Premise: with the OS page cache warm, a key lookup served from +// MDBX vs from a snapshot .kv file should cost roughly the same — the +// disk and page cache are identical. Today they don't. These benches +// quantify the gap (H0) and decompose it (H1-H4). +// +// **Status: SCAFFOLD.** Compiles and runs (skipped) so the structure +// survives. Real-datadir bootstrap and synthetic-fixture bootstrap are +// the two TODOs that turn this into a measurement tool. +// +// Two operating modes are anticipated: +// +// 1. **Synthetic mode** (preferred for CI / reproducibility): +// use testDbAndAggregatorBench, write K1 keys at low txNums, +// buildFiles+prune so they land in files, write K2 at high +// txNums and leave in MDBX. No external datadir required. +// +// 2. **Real-datadir mode** (preferred for fidelity): +// open an existing perf-devnet-3 / mainnet datadir read-only, +// pick keys by cursor-iterating the values table (mdbxKeys) and +// walking the latest .kv decompressor (fileKeys). Matches +// production file-count, Bloom false-positive rate, etc. +// +// Both fail with b.Skip() in this scaffold; the TODOs below are the +// agenda for turning it into a real measurement. + +import ( + "encoding/binary" + "flag" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/length" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state" + "github.com/erigontech/erigon/db/state/execctx" +) + +var snapDataDir = flag.String("snapdatadir", "", + "path to a real datadir (chaindata + snapshots) for snapshot-vs-mdbx benches; "+ + "when empty, real-datadir benches are skipped and synthetic mode runs") + +// snapBenchSetup carries everything a sub-bench needs: an open RO tx, +// the AggregatorRoTx for Debug* paths, the domain under test, and the +// pre-picked key sets. Set up once per Benchmark function so warmup +// and timing happen on the same data. +type snapBenchSetup struct { + tx kv.TemporalTx + aggTx *state.AggregatorRoTx + domain kv.Domain + mdbxKeys [][]byte + fileKeys [][]byte +} + +// openSnapBench builds a snapBenchSetup for the given domain. Picks +// real-datadir mode iff --snapdatadir is set, otherwise synthetic. +// Both paths currently call b.Skip; the TODOs below are the work. +func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { + b.Helper() + if *snapDataDir != "" { + return openSnapBenchRealDatadir(b, domain, *snapDataDir) + } + return openSnapBenchSynthetic(b, domain) +} + +// openSnapBenchRealDatadir opens an existing chaindata+snapshots +// datadir read-only and picks keys from production files. +// +// TODO: wire this up. Open via the same path turbo/node uses: open +// MDBX RwDB, open *state.Aggregator, wrap via temporal.New, begin RO +// tx, type-assert tx.AggTx() to *AggregatorRoTx. Then call pickKeys. +func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadir string) *snapBenchSetup { + b.Helper() + b.Skip("TODO: real-datadir bootstrap (open MDBX + Aggregator + temporal.DB " + + "the same way cmd/erigon does, then BeginTemporalRo)") + return nil +} + +// openSnapBenchSynthetic builds an in-memory aggregator with a known +// key distribution and returns a setup whose two key sets are +// guaranteed to live on the file path and the MDBX path respectively. +// +// Sequence: +// +// 1. testDbAndAggregatorBench builds a fresh agg + tempdir. +// 2. Phase 1 — write keysetSize "old" keys at txNums spanning steps +// [0..stepsToFlush-1], deterministically generated from seed +// phase1Seed so the same bench reproduces. +// 3. Flush + commit + agg.BuildFiles(stepsToFlush*aggStep) so those +// keys land in .kv files. PruneSmallBatches removes them from +// MDBX. After this, phase-1 keys are file-resident. +// 4. Phase 2 — write keysetSize "new" keys at txNums spanning steps +// [stepsToFlush..stepsToFlush+1], from disjoint phase2Seed so no +// overlap with phase 1. NO BuildFiles for these. They stay in +// MDBX. +// 5. Begin a RO temporal tx, return setup. +// +// We verify each key actually lives where we expect via +// DebugGetLatestFromDB / DebugGetLatestFromFiles — fail loudly if a +// key isn't where the test plan says it should be (catches setup +// regressions). +func openSnapBenchSynthetic(b *testing.B, domain kv.Domain) *snapBenchSetup { + b.Helper() + + const ( + // aggStep small + many full steps mirrors how the fuzz test sets + // up data. BuildFiles only builds completed steps, so phase-1 has + // to span an integer number of full steps. + aggStep = 16 + stepsToFlush = 64 // phase 1 spans this many full steps + keysetSize = stepsToFlush * aggStep // = 1024; one key per txNum, every step is full + phase1Seed = 0xC0FFEE + phase2Seed = 0xDECAFB + // phase-2 starts well past the built step boundary so its keys + // can't be lumped into a future build. + phase2BaseTxNum = uint64(stepsToFlush) * aggStep + ) + + // keySize varies per domain; values are 8-byte counters (txNum + // encoded big-endian) so put/lookup paths are exercised. + var keySize int + switch domain { + case kv.AccountsDomain: + keySize = length.Addr + case kv.StorageDomain: + keySize = length.Addr + length.Hash + default: + b.Fatalf("openSnapBenchSynthetic: domain %v not yet wired", domain) + } + + db, agg := testDbAndAggregatorBench(b, aggStep) + logger := log.New() + + // ---- Phase 1: keys destined for files. Each key gets its own + // txNum so per-tx history entries are well-formed. + phase1Keys := genKeys(phase1Seed, keysetSize, keySize) + + { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + + domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) + require.NoError(b, err) + + val := make([]byte, 8) + for i, k := range phase1Keys { + // One key per txNum, packing every txNum in the + // stepsToFlush * aggStep range exactly once. This guarantees + // every step is full so BuildFiles emits a .kv per step. + txNum := uint64(i) + binary.BigEndian.PutUint64(val, txNum+1) // +1 so all values are non-zero + err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) + require.NoError(b, err) + } + + // ComputeCommitment writes the trie root into the CommitmentDomain + // so a follow-up NewSharedDomains can SeekCommitment without + // erroring "commitment state out of date". + lastTxNum := uint64(keysetSize - 1) + _, err = domains.ComputeCommitment(b.Context(), rwTx, true /*save*/, 0, lastTxNum, "h0-bench", nil) + require.NoError(b, err) + + require.NoError(b, domains.Flush(b.Context(), rwTx)) + domains.Close() + require.NoError(b, rwTx.Commit()) + } + + // Build files for the phase-1 range. + require.NoError(b, agg.BuildFiles(uint64(stepsToFlush)*aggStep)) + + // Prune phase-1 entries out of MDBX so they're file-only. + // PruneSmallBatches may report haveMore=true when batch limits stop + // it short of fully draining; loop until drained or we hit a sane + // safety bound. time.Hour keeps it in "furious" prune mode (large + // per-iteration limit), so this is fast in practice. + for round := 0; round < 32; round++ { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + haveMore, err := rwTx.PruneSmallBatches(b.Context(), time.Hour) + require.NoError(b, err) + require.NoError(b, rwTx.Commit()) + if !haveMore { + break + } + } + + // ---- Phase 2: keys destined for MDBX (no BuildFiles after). + phase2Keys := genKeys(phase2Seed, keysetSize, keySize) + + { + rwTx, err := db.BeginTemporalRw(b.Context()) + require.NoError(b, err) + + domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) + require.NoError(b, err) + + val := make([]byte, 8) + for i, k := range phase2Keys { + // Phase-2 keys live well past the built-step boundary so a + // hypothetical follow-up BuildFiles wouldn't sweep them in. + txNum := phase2BaseTxNum + uint64(i) + binary.BigEndian.PutUint64(val, txNum+1) + err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) + require.NoError(b, err) + } + + require.NoError(b, domains.Flush(b.Context(), rwTx)) + domains.Close() + require.NoError(b, rwTx.Commit()) + } + + // ---- Open a RO tx for benching. + tx, err := db.BeginTemporalRo(b.Context()) + require.NoError(b, err) + b.Cleanup(func() { tx.Rollback() }) + + aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) + require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") + + // Partition each phase set by actual residency. We expect: + // phase 1 -> almost all "file", with a tail of tip-step keys still + // in MDBX because Prune retains the most-recent step; + // phase 2 -> all "mdbx" (no BuildFiles called for it). + // The bench uses only the keys that landed where we want, so any + // misroute just shrinks the keyset rather than corrupting timings. + mdbxFromP2, fileFromP2, missingP2 := partitionByResidency(b, aggTx, tx, domain, phase2Keys) + mdbxFromP1, fileFromP1, missingP1 := partitionByResidency(b, aggTx, tx, domain, phase1Keys) + if missingP1 > 0 || missingP2 > 0 { + b.Fatalf("residency partition: missing keys p1=%d p2=%d", missingP1, missingP2) + } + b.Logf("residency: phase1 file=%d mdbx=%d phase2 file=%d mdbx=%d", + len(fileFromP1), len(mdbxFromP1), len(fileFromP2), len(mdbxFromP2)) + + mdbxKeys := mdbxFromP2 // bench's mdbx-resident set + fileKeys := fileFromP1 // bench's file-resident set + require.NotEmpty(b, mdbxKeys, "synthetic fixture produced no MDBX-resident keys") + require.NotEmpty(b, fileKeys, "synthetic fixture produced no file-resident keys") + + return &snapBenchSetup{ + tx: tx, + aggTx: aggTx, + domain: domain, + mdbxKeys: mdbxKeys, + fileKeys: fileKeys, + } +} + +// genKeys produces n deterministic keys of size keySize bytes from +// the given seed. Same seed -> same keys -> reproducible benches. +func genKeys(seed uint64, n, keySize int) [][]byte { + rnd := newRnd(seed) + keys := make([][]byte, n) + for i := range keys { + k := make([]byte, keySize) + _, _ = rnd.Read(k) + keys[i] = k + } + return keys +} + +// partitionByResidency walks keys and groups them by which read path +// returns the value: "mdbx" (DebugGetLatestFromDB ok), "file" +// (DebugGetLatestFromDB miss + DebugGetLatestFromFiles ok), or +// "missing" (neither). missing is reported as a count for the caller +// to fail on — no key in our synthetic setup should be unreachable. +func partitionByResidency(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, keys [][]byte) (mdbxKeys, fileKeys [][]byte, missing int) { + b.Helper() + for _, k := range keys { + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, k, tx) + require.NoError(b, err) + if dbOk { + mdbxKeys = append(mdbxKeys, k) + continue + } + _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, k, 0) + require.NoError(b, err) + if fileOk { + fileKeys = append(fileKeys, k) + continue + } + missing++ + } + return +} + +// warmup reads every key in keys via tx.GetLatest so the OS page cache +// is hot for all relevant snapshot/MDBX pages before timing starts. +// Required because the H0 claim only holds for warm pages — we want to +// measure software overhead, not disk I/O. +func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { + for _, k := range keys { + _, _, _ = tx.GetLatest(domain, k) + } +} + +// runH0 is the H0 measurement body. Four sub-benches: +// +// - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb +// and returns; never touches files. Baseline cost. +// +// - File_path: tx.GetLatest with key only in files. Misses MDBX, +// then walks getLatestFromFiles (Bloom -> recsplit -> seg +// decompress). Baseline-with-files cost. +// +// - Forced_file_path: same MDBX-resident key set as MDBX_path but +// routed through DebugGetLatestFromFiles, forcing the file path +// even though the key would have hit MDBX. Isolates per-key +// file-path cost from key-distribution effects. +// +// - Forced_db_path: DebugGetLatestFromDB on the MDBX key set — +// mirror of Forced_file_path, isolates the MDBX-only path cost. +// +// Headline: file_ns_per_op / mdbx_ns_per_op. Forced_* sub-benches +// confirm the ratio isn't a key-set artifact. +func runH0(b *testing.B, setup *snapBenchSetup) { + b.Helper() + + // Two warmup passes: first lands pages in the OS page cache, + // second absorbs L3/TLB effects so we time steady-state. + warmup(setup.tx, setup.domain, setup.mdbxKeys) + warmup(setup.tx, setup.domain, setup.fileKeys) + warmup(setup.tx, setup.domain, setup.mdbxKeys) + warmup(setup.tx, setup.domain, setup.fileKeys) + + b.Run("MDBX_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) + } + }) + + b.Run("File_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) + } + }) + + // Forced_file_path: file-resident keys via the file-only debug path. + // Strips MDBX-miss cost from File_path so we see pure file-side + // work (Bloom -> recsplit -> seg decompress). + b.Run("Forced_file_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( + setup.domain, setup.fileKeys[i%len(setup.fileKeys)], 0) + } + }) + + // Forced_db_path: MDBX-resident keys via the DB-only debug path. + // Strips routing/dispatch overhead from MDBX_path so we see pure + // MDBX cursor work. + b.Run("Forced_db_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( + setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], setup.tx) + } + }) + + // Bloom_miss_path: MDBX-resident keys against the file-only debug + // path. These keys are NOT in any .kv file so each lookup is a + // pure xorfilter "not present" probe per file. Gives the Bloom-miss + // cost in isolation — useful for H1 (per-file Bloom probe overhead). + b.Run("Bloom_miss_path", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( + setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], 0) + } + }) +} + +// BenchmarkSnapVsMDBX_H0_Accounts measures the warm-cache file-vs-MDBX +// per-key gap on the AccountsDomain. Headline: file_ns/op : mdbx_ns/op. +func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { + runH0(b, openSnapBench(b, kv.AccountsDomain)) +} + +// BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. +// Storage dominates file-read traffic on bloat workloads (per +// runs-step9-cache-behind-sd memory), so getting the gap for both +// Accounts and Storage is the minimum useful H0 output. +func BenchmarkSnapVsMDBX_H0_Storage(b *testing.B) { + runH0(b, openSnapBench(b, kv.StorageDomain)) +} From 118298ac94d83e3ebd66fc451beb3f82fe6dc89d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 17:15:44 +0000 Subject: [PATCH 028/120] db/state: H0 real-datadir bootstrap; first production numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the --snapdatadir flag path to the H0 bench. Opens an existing chaindata + snapshots datadir read-only via the same recipe as cmd/integration (mdbx Accede + state.New + temporal.New) and picks keys by cursor-walking the per-domain values table. Pragmatic adjustments: - On heavily-pruned production datadirs (perf-devnet-3-run was 100% pruned), every MDBX values-table row is step-shadowed by a file, so getLatestFromDb returns ok=false. The MDBX-side sub-benches skip in this case; File_path numbers stand on their own and the synthetic MDBX_path baseline serves as the cross- mode comparator. - skipIfEmpty short-circuits per sub-bench rather than failing the whole run, so we can still get the file-path numbers. First production numbers (AMD EPYC 4244P, AccountsDomain, 2012 file-resident keys from perf-devnet-3-run, fully pruned): File_path 211 ns/op (synthetic was 226; essentially same) Forced_file_path 30 ns/op (synthetic was 30; identical) Surprising finding: real .kv file reads cost the same as synthetic. This means production bloat-workload bottleneck is NOT in getLatestFromFiles — it must be in HistorySeek (.ef history files walked by HistoryStateReader.GetAsOf). The GetAsOf shortcut work flagged in getasof-regression-suspect.md is the right lead. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 212 +++++++++++++++++++++++- 1 file changed, 205 insertions(+), 7 deletions(-) diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go index 8d431a046df..c838a1a8d4c 100644 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -54,7 +54,11 @@ import ( "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/mdbx" + "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/execctx" ) @@ -89,14 +93,182 @@ func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { // openSnapBenchRealDatadir opens an existing chaindata+snapshots // datadir read-only and picks keys from production files. // -// TODO: wire this up. Open via the same path turbo/node uses: open -// MDBX RwDB, open *state.Aggregator, wrap via temporal.New, begin RO -// tx, type-assert tx.AggTx() to *AggregatorRoTx. Then call pickKeys. -func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadir string) *snapBenchSetup { +// Uses MDBX Accede mode (don't create, must already exist) and +// state.New(...).MustOpen which loads the on-disk schema. ResolveDB +// settings reads stepSize / stepsInFrozenFile from the existing DB. +// +// CAUTION: opening a chaindata that another erigon process is writing +// to is risky. Bench against an idle copy. The script that drives +// these benches typically does prepare-run.sh first to snapshot a +// fresh chaindata copy under /erigon-data/perf-devnet-3-run/ or +// similar. Pass that path via --snapdatadir. +func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadirPath string) *snapBenchSetup { + b.Helper() + logger := log.New() + dirs := datadir.New(datadirPath) + + rawDB := mdbx.New(dbcfg.ChainDB, logger). + Path(dirs.Chaindata). + Accede(true). // must exist; do not create + MustOpen() + b.Cleanup(rawDB.Close) + + settings, err := state.ResolveErigonDBSettings(dirs, logger, false /*noDownloader*/) + require.NoError(b, err, "ResolveErigonDBSettings") + + agg := state.New(dirs). + Logger(logger). + WithErigonDBSettings(settings). + MustOpen(b.Context(), rawDB) + require.NoError(b, agg.OpenFolder(), "agg.OpenFolder") + b.Cleanup(agg.Close) + + tdb, err := temporal.New(rawDB, agg) + require.NoError(b, err) + b.Cleanup(tdb.Close) + + tx, err := tdb.BeginTemporalRo(b.Context()) + require.NoError(b, err) + b.Cleanup(func() { tx.Rollback() }) + + aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) + require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") + + mdbxKeys, fileKeys := pickRealDatadirKeys(b, aggTx, tx, domain, 10_000) + require.NotEmpty(b, fileKeys, "no file-resident keys found in %s for domain %v", datadirPath, domain) + if len(mdbxKeys) == 0 { + // Heavily pruned production datadir — every MDBX row is + // step-shadowed by a file. The MDBX-side sub-benches will skip + // (runH0 short-circuits on empty key set). Compare File_path + // against the synthetic MDBX_path baseline; clearly note the + // cross-mode caveat in the issue write-up. + b.Logf("real-datadir: no MDBX-resident keys (datadir is fully pruned). " + + "File_path numbers are real; cross-reference MDBX_path against the synthetic bench.") + } + b.Logf("real-datadir keys: mdbx=%d file=%d (domain=%v, path=%s)", + len(mdbxKeys), len(fileKeys), domain, datadirPath) + + return &snapBenchSetup{ + tx: tx, + aggTx: aggTx, + domain: domain, + mdbxKeys: mdbxKeys, + fileKeys: fileKeys, + } +} + +// pickRealDatadirKeys walks the existing datadir to find up to +// `target` MDBX-resident and `target` file-resident keys for the +// given domain. +// +// Strategy A — cursor-walk the per-domain values table for +// MDBX-resident keys (those rows ARE in MDBX by definition). Strip +// the trailing 8-byte inverted-step from each row's key. +// +// Strategy B — for file-resident keys, randomly sample address-sized +// or storage-key-sized byte strings from the cursor walk's MDBX keys +// (deterministic via seeded RNG): for each candidate, query +// DebugGetLatestFromFiles. If the file path returns ok and MDBX does +// NOT, the key is file-resident. This is approximate — keys that +// exist BOTH in MDBX and files (during prune transitions) are +// classified as MDBX-resident, which is what we want for bench +// timing accuracy (MDBX always wins on getLatest). +// +// Practical note: in production datadirs the MDBX values table +// usually has fewer rows than the snapshot files combined, so we +// expect mdbxKeys to fill faster than fileKeys. To find file-only +// keys we'd need to walk the .kv files directly via seg.Decompressor +// — that's a follow-on once Strategy B is shown to be insufficient. +func pickRealDatadirKeys(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, target int) (mdbxKeys, fileKeys [][]byte) { + b.Helper() + table := domainValsTable(b, domain) + + c, err := tx.Cursor(table) + require.NoError(b, err) + defer c.Close() + + const stepSuffixLen = 8 // values-table key = realKey || invertedStep + seen := make(map[string]struct{}) + var rowsScanned, dbOkCount int + + // First pass: collect MDBX-resident keys directly from the values table. + for k, _, err := c.First(); k != nil; k, _, err = c.Next() { + require.NoError(b, err) + rowsScanned++ + if len(k) <= stepSuffixLen { + continue // unexpected; skip + } + realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) + ks := string(realKey) + if _, dup := seen[ks]; dup { + continue + } + seen[ks] = struct{}{} + + // Confirm MDBX residency via DebugGetLatestFromDB so we don't + // include keys that, despite having a row here, don't actually + // resolve in the read path (edge cases around step bounds). + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) + require.NoError(b, err) + if dbOk { + dbOkCount++ + mdbxKeys = append(mdbxKeys, realKey) + if len(mdbxKeys) >= target { + break + } + } + } + b.Logf("first-pass scan: table=%s rows=%d unique_keys=%d dbOk=%d", + table, rowsScanned, len(seen), dbOkCount) + + // Second pass: walk the same cursor again looking for keys that + // MISS MDBX but HIT files. Iterating the values table catches + // keys that left a row but have all data in files (delete + // markers, post-prune residue), which is approximate but cheap. + // For a full file-only walk we'd open the .kv decompressor; this + // is good enough for the H0 first cut. + for k, _, err := c.First(); k != nil; k, _, err = c.Next() { + if len(fileKeys) >= target { + break + } + require.NoError(b, err) + if len(k) <= stepSuffixLen { + continue + } + realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) + + _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) + require.NoError(b, err) + if dbOk { + continue // MDBX already serves this; not file-only + } + _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, realKey, 0) + require.NoError(b, err) + if fileOk { + fileKeys = append(fileKeys, realKey) + } + } + + return mdbxKeys, fileKeys +} + +// domainValsTable returns the MDBX values-table name for the given +// domain. Used by pickRealDatadirKeys to cursor-walk MDBX directly. +func domainValsTable(b *testing.B, domain kv.Domain) string { b.Helper() - b.Skip("TODO: real-datadir bootstrap (open MDBX + Aggregator + temporal.DB " + - "the same way cmd/erigon does, then BeginTemporalRo)") - return nil + switch domain { + case kv.AccountsDomain: + return kv.TblAccountVals + case kv.StorageDomain: + return kv.TblStorageVals + case kv.CodeDomain: + return kv.TblCodeVals + case kv.CommitmentDomain: + return kv.TblCommitmentVals + default: + b.Fatalf("domainValsTable: domain %v not mapped", domain) + return "" + } } // openSnapBenchSynthetic builds an in-memory aggregator with a known @@ -316,6 +488,17 @@ func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { } } +// skipIfEmpty short-circuits the calling sub-bench when the key set +// is empty (real-datadir mode often has no MDBX-resident keys after +// aggressive pruning). +func skipIfEmpty(b *testing.B, name string, keys [][]byte) bool { + if len(keys) == 0 { + b.Skipf("no keys for %s sub-bench", name) + return true + } + return false +} + // runH0 is the H0 measurement body. Four sub-benches: // // - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb @@ -346,6 +529,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { warmup(setup.tx, setup.domain, setup.fileKeys) b.Run("MDBX_path", func(b *testing.B) { + if skipIfEmpty(b, "MDBX_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) @@ -353,6 +539,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { }) b.Run("File_path", func(b *testing.B) { + if skipIfEmpty(b, "File_path", setup.fileKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) @@ -363,6 +552,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // Strips MDBX-miss cost from File_path so we see pure file-side // work (Bloom -> recsplit -> seg decompress). b.Run("Forced_file_path", func(b *testing.B) { + if skipIfEmpty(b, "Forced_file_path", setup.fileKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( @@ -374,6 +566,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // Strips routing/dispatch overhead from MDBX_path so we see pure // MDBX cursor work. b.Run("Forced_db_path", func(b *testing.B) { + if skipIfEmpty(b, "Forced_db_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( @@ -386,6 +581,9 @@ func runH0(b *testing.B, setup *snapBenchSetup) { // pure xorfilter "not present" probe per file. Gives the Bloom-miss // cost in isolation — useful for H1 (per-file Bloom probe overhead). b.Run("Bloom_miss_path", func(b *testing.B) { + if skipIfEmpty(b, "Bloom_miss_path", setup.mdbxKeys) { + return + } b.ReportAllocs() for i := 0; b.Loop(); i++ { _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( From 005790f800e159f2b869a05cbfa50fcad162e930 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 7 May 2026 21:42:03 +0000 Subject: [PATCH 029/120] db/state, commitment: H_GetAsOf bench + per-block calculator timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related additions to the snapshot-vs-MDBX perf-equivalence investigation (memory: snapshot-vs-mdbx-performance-equivalence.md): 1. H_GetAsOf bench (db/state/snapshot_vs_mdbx_bench_test.go). New runHGetAsOf with four sub-benches for HistorySeek-via-GetAsOf on file-resident keys: GetLatest_baseline, GetAsOf_recent (asOf near endTxNum), GetAsOf_mid (asOf at endTxNum/2), GetAsOf_floor (asOf=1). Tests the path the calculator's HistoryStateReader.Read uses, which is distinct from getLatestFromFiles measured in H0. Real-datadir results on perf-devnet-3 (AccountsDomain, endTxNum=2.9B): GetLatest_baseline 202 ns/op 0 allocs GetAsOf_recent 570 ns/op 0 allocs <- 2.8x baseline, no result GetAsOf_mid 235 ns/op 5 allocs GetAsOf_floor 196 ns/op 4 allocs GetAsOf_recent (the calculator's pattern after PR #21010) scans the .ef looking for a record at-or-after endTxNum-1, finds none (most keys haven't changed in the last txNum), falls through to GetLatest. The 370ns/op overhead vs GetLatest is wasted scan. Confirms the GetAsOf shortcut described in getasof-regression-suspect.md as a real lever, though small in absolute terms (~2ms/block on the bloat workload). 2. Surface "took" + "keys" on the existing [commitment][cache-fp] Info log line (commitmentdb). Was already computed in the debug-level "[commitment] processed" log, but the bench runs with --log.dir.disable so debug logs aren't captured. This made it possible to attribute the 4.3s gap inside newPayload(TEST block) directly: the calculator's ComputeCommitment takes 4220ms for the 5910-key bloat block — 91% of the entire block wall time. Per-key cost is ~700us, consistent across blocks of all sizes. The actual perf lever for the bloat workload is making per-branch ComputeCommitment cheaper, not file/state reads. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/snapshot_vs_mdbx_bench_test.go | 107 ++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go index c838a1a8d4c..c2efacdcab9 100644 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ b/db/state/snapshot_vs_mdbx_bench_test.go @@ -598,6 +598,113 @@ func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { runH0(b, openSnapBench(b, kv.AccountsDomain)) } +// runHGetAsOf measures HistorySeek-via-GetAsOf cost on file-resident +// keys. The motivation (per H0 finding): `getLatestFromFiles` is fast +// (~30 ns isolated); the bloat-workload pprof showed real file-read +// pressure that this can't account for. The calculator's +// HistoryStateReader.Read calls tx.GetAsOf, which goes through +// HistorySeek and walks .ef history files — a different code path +// than getLatestFromFiles. +// +// Sub-benches: +// +// - GetLatest_baseline: tx.GetLatest on file-resident keys; mirrors +// H0's File_path so we can confirm the same setup against a fresh +// comparator. +// +// - GetAsOf_recent: tx.GetAsOf at asOfTxNum = endTxNum - 1. Most +// keys won't have a history record post-asOf, so HistorySeek +// short-circuits to the latest path. Lower bound on GetAsOf cost. +// +// - GetAsOf_mid: tx.GetAsOf at asOfTxNum = endTxNum / 2. Forces +// the .ef walk to seek through more history. This is the cost +// shape the calculator pays when reading historic state. +// +// - GetAsOf_zero: tx.GetAsOf at asOfTxNum = HistoryStartFrom() + 1 +// (just past the visible window start). Maximally adversarial — +// full .ef walk depth. The calculator's old asOfReader.txNum=0 +// bug (PR #21010) hit this path when the window started past 0. +// +// All sub-benches use the same fileKeys set so per-key cost is +// directly comparable across the four. +func runHGetAsOf(b *testing.B, setup *snapBenchSetup) { + b.Helper() + if skipIfEmpty(b, "H_GetAsOf", setup.fileKeys) { + return + } + + // Snapshot the visible-history window so all four sub-benches use + // the same anchors and so we report them in the bench log. We + // don't probe historyStartFrom from here (no public accessor on + // AggregatorRoTx for per-domain history range); instead use simple + // fixed fractions of endTxNum. Adjust asOfFloor=1 to avoid + // hitting txNum=0 which can error on snapshot-loaded chains + // (PR #21010 was that bug; we don't want H_GetAsOf to trip on it). + endTxNum := setup.aggTx.EndTxNumNoCommitment() + asOfRecent := endTxNum + if endTxNum > 0 { + asOfRecent = endTxNum - 1 + } + asOfMid := endTxNum / 2 + asOfFloor := uint64(1) + b.Logf("H_GetAsOf anchors: endTxNum=%d -> asOfRecent=%d asOfMid=%d asOfFloor=%d", + endTxNum, asOfRecent, asOfMid, asOfFloor) + + // Two warmup passes so the OS page cache holds the .ef files we + // care about. Touch each anchor txNum so the relevant history + // segments are mmap'd in. + for _, asOf := range []uint64{asOfRecent, asOfMid, asOfFloor} { + for _, k := range setup.fileKeys { + _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) + } + for _, k := range setup.fileKeys { + _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) + } + } + + b.Run("GetLatest_baseline", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) + } + }) + + b.Run("GetAsOf_recent", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfRecent) + } + }) + + b.Run("GetAsOf_mid", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfMid) + } + }) + + b.Run("GetAsOf_floor", func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfFloor) + } + }) +} + +// BenchmarkSnapVsMDBX_HGetAsOf_Accounts measures HistorySeek cost on +// real file-resident keys via tx.GetAsOf. Confirms the H0 finding +// that the production bloat-workload bottleneck is in .ef history +// walking, not .kv latest-state reads. +func BenchmarkSnapVsMDBX_HGetAsOf_Accounts(b *testing.B) { + runHGetAsOf(b, openSnapBench(b, kv.AccountsDomain)) +} + +// BenchmarkSnapVsMDBX_HGetAsOf_Storage — same for StorageDomain. +// Storage tends to dominate in the bloat workload. +func BenchmarkSnapVsMDBX_HGetAsOf_Storage(b *testing.B) { + runHGetAsOf(b, openSnapBench(b, kv.StorageDomain)) +} + // BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. // Storage dominates file-read traffic on bloat workloads (per // runs-step9-cache-behind-sd memory), so getting the gap for both diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 88fab39952c..9f84b34b057 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -344,6 +344,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // so two builds running the same workload can be diffed offline to // localise the first block at which their caches diverge. See // commitment.BranchCache.Fingerprint for the hash semantics. + // took + keys exposed at Info so the calculator's per-block compute + // time is visible without enabling debug logs (used by the + // snapshot-vs-mdbx perf-equivalence investigation to attribute + // the gap inside newPayload). if logCacheFingerprint { if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { if bc := hph.BranchCache(); bc != nil { @@ -351,7 +355,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "block", blockNum, "root", hex.EncodeToString(rootHash), "fp", fmt.Sprintf("%016x", bc.Fingerprint()), - "divergences", bc.VerifyDivergences()) + "divergences", bc.VerifyDivergences(), + "took", took, + "keys", common.PrettyCounter(updateCount)) } } } From 4276219f3a9397d21ab289a9a6b4bd7891426d96 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 08:51:40 +0000 Subject: [PATCH 030/120] commitment: surface per-block telemetry on cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three groups of fields to the existing [commitment][cache-fp] log line so the calculator's per-block behaviour is observable at Info level (the bench runs with --log.dir.disable, so debug logs aren't captured): - took, keys: ComputeCommitment wall + key-count for the block. Pre-existing internally, now surfaced. - load, skipped, reset: process-cumulative counts of computeCellHash decisions: * load = had no memoized stateHash, fetched value from DB * skipped = had memoized stateHash, reused without fetch * reset = had stateHash but had to invalidate it Surfaced via new commitment.SkipLoadResetCounters(). - files_acc / files_sto / files_code / files_comm: per-domain file-read counts pulled from sd.Metrics().Domains[domain]. Decomposes the aggregate `files=N` from the [domain reads] log line into its actual sources (e.g. on the SSTORE-bloated block the 32k file reads break down as 5.9k Storage value loads + 26.6k Commitment branch reads + a handful of others). All counters are cumulative; per-block deltas are obtained by subtracting consecutive cache-fp lines. Pure observability — no behaviour change. Used as the measurement framework for the snapshot-vs-MDBX perf-equivalence investigation and the follow-on commits that target specific levers. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../commitmentdb/commitment_context.go | 44 ++++++++++++++++++- execution/commitment/hex_patricia_hashed.go | 19 ++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 9f84b34b057..742a92ce0b8 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -21,6 +21,7 @@ import ( "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" + "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" @@ -53,6 +54,13 @@ type sd interface { // to localise which state layer holds bytes that diverge from // cache. Read-only. ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) + + // Metrics exposes the per-SD DomainMetrics so callers can read + // per-domain (cache, db, file) read counters. Used by the + // cache-fp log line to break the aggregate `files=N` count down + // per domain (Storage value loads vs Commitment branch reads + // vs Account loads). + Metrics() *changeset.DomainMetrics } type SharedDomainsCommitmentContext struct { @@ -351,13 +359,47 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context if logCacheFingerprint { if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { if bc := hph.BranchCache(); bc != nil { + // Skip/load/reset counters are process-cumulative; the + // per-block delta requires subtracting consecutive lines. + // Used to answer: "what fraction of computeCellHash calls + // actually fetched the underlying value vs reused a + // memoized stateHash?" + load, skipped, reset := commitment.SkipLoadResetCounters() + // Per-domain file-read counts: lets us decompose the + // aggregate `files=N` from [domain reads] into Commitment + // (branch reads), Storage (value loads), Account, Code. + // All cumulative; deltas via successive lines. + var aFiles, sFiles, cFiles, mFiles int64 + if m := sdc.sharedDomains.Metrics(); m != nil { + m.RLock() + if d := m.Domains[kv.AccountsDomain]; d != nil { + aFiles = d.FileReadCount + } + if d := m.Domains[kv.StorageDomain]; d != nil { + sFiles = d.FileReadCount + } + if d := m.Domains[kv.CodeDomain]; d != nil { + cFiles = d.FileReadCount + } + if d := m.Domains[kv.CommitmentDomain]; d != nil { + mFiles = d.FileReadCount + } + m.RUnlock() + } log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), "fp", fmt.Sprintf("%016x", bc.Fingerprint()), "divergences", bc.VerifyDivergences(), "took", took, - "keys", common.PrettyCounter(updateCount)) + "keys", common.PrettyCounter(updateCount), + "load", load, + "skipped", skipped, + "reset", reset, + "files_acc", aFiles, + "files_sto", sFiles, + "files_code", cFiles, + "files_comm", mFiles) } } } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 3f53e71fa04..5ac89fd8ae9 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1871,6 +1871,25 @@ var ( hadToReset atomic.Uint64 ) +// SkipLoadResetCounters returns the cumulative process-wide counts of: +// - hadToLoad: computeCellHash had no memoized stateHash and had +// to fetch the underlying value from cache/DB to compute the leaf +// hash. This is the real "did we go to disk for state we needed" +// count. +// - skippedLoad: the cell had a memoized stateHash so the fetch was +// skipped entirely (mxTrieStateSkipRate also counts these). +// - hadToReset: the cell HAD a memoized stateHash but it was +// invalidated (extension fold path, depth change). Implies a +// fresh hash compute and may imply a load. +// +// Counters are atomic and process-cumulative. To get per-block +// deltas, snapshot before/after the block. Used by the perf-equivalence +// investigation to measure whether memoization is doing its job or +// whether we're paying for fetches that should have been skipped. +func SkipLoadResetCounters() (load, skipped, reset uint64) { + return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load() +} + type skipStat struct { accLoaded, accSkipped, accReset, storReset, storLoaded, storSkipped uint64 } From 1e8c25c58768c1ab2556f799957a57c27039aea5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 08:58:47 +0000 Subject: [PATCH 031/120] stagedsync/calc_state: skip wasted ensureStorage in StoragePath ApplyWrites cs.ensureStorage(addr, key) lazy-loads the current value via cs.domainReader.ReadAccountStorage, which goes through GetAsOf and walks .ef history files. On the StoragePath case, the loaded value is *immediately overwritten* on the next line by the EVM-write value (cs.storageState[addr][key] = v). The lazy-load is therefore wasted: a cold .ef seek per first-touched slot, whose result is discarded. On the SSTORE-bloated TEST block (5,910 storage writes), this is ~5,910 wasted GetAsOf calls per block. Replace with bare inner-map initialization so storageState[addr] is non-nil before the assignment. The downstream consumer (FlushToUpdates) only reads the value we just set (the EVM write), so dropping the prior value is correct. Other callers of ensureStorage (Account etc.) are unchanged. Tested: stagedsync test suite passes. Per-block telemetry on the [commitment][cache-fp] log (commit aa424f01fe) will show the 'load' delta drop on benches that hit StoragePath writes. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/stagedsync/calc_state.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/execution/stagedsync/calc_state.go b/execution/stagedsync/calc_state.go index d22781445fe..9a14af64afe 100644 --- a/execution/stagedsync/calc_state.go +++ b/execution/stagedsync/calc_state.go @@ -296,8 +296,19 @@ func (cs *calcState) ApplyWrites(writes state.VersionedWrites) { } case state.StoragePath: v := w.Val.(uint256.Int) - cs.ensureStorage(w.Address, w.Key) // lazy-load if needed - cs.storageState[w.Address][w.Key] = v + // The previous slot value is irrelevant here: the next line + // overwrites it with the EVM-write value, and the only + // downstream consumer (FlushToUpdates) reads exactly the + // value we set. Skip the ensureStorage lazy-load — it + // triggers a cold .ef GetAsOf seek per first-touched slot + // (~5,910 wasted seeks per SSTORE-bloat block) and discards + // the loaded value. Initialize just the inner map. + slots := cs.storageState[w.Address] + if slots == nil { + slots = make(map[accounts.StorageKey]uint256.Int) + cs.storageState[w.Address] = slots + } + slots[w.Key] = v if cs.storageDirty[w.Address] == nil { cs.storageDirty[w.Address] = make(map[accounts.StorageKey]bool) } From a26369503f6ae8301be0f9698fa5deba3be44936 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 09:17:01 +0000 Subject: [PATCH 032/120] commitment: split disk-load counter from EVM-write plumbing setFromUpdate increments hadToLoad on every call, conflating two distinct paths: (1) trie went to *FromCacheOrDB during computeCellHash because it didn't have the value, and (2) the executor plumbed an EVM-write through Updates into the cell. Both are valid uses but only (1) is the disk-fetch we're trying to eliminate. Add diskLoadStorage / diskLoadAccount that increment ONLY at the three *FromCacheOrDB call sites in computeCellHash, and surface them on the [commitment][cache-fp] log line as disk_sto / disk_acc. With this split, the bench output tells us directly whether eliminating the storageFromCacheOrDB path (C3 plumb-EVM-value-to-cell) is worth doing on the bloat workload, or whether the load count is dominated by EVM-write plumbing and the lever is elsewhere. No behaviour change; counters are atomic and gated by the existing BRANCH_CACHE_FINGERPRINT log path. --- .../commitmentdb/commitment_context.go | 4 ++- execution/commitment/hex_patricia_hashed.go | 26 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 742a92ce0b8..c43775fe2d8 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -364,7 +364,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // Used to answer: "what fraction of computeCellHash calls // actually fetched the underlying value vs reused a // memoized stateHash?" - load, skipped, reset := commitment.SkipLoadResetCounters() + load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -396,6 +396,8 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "load", load, "skipped", skipped, "reset", reset, + "disk_sto", diskSto, + "disk_acc", diskAcc, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 5ac89fd8ae9..1e6080e80a5 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1016,6 +1016,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } else { if !cell.loaded.storage() { hph.metrics.StorageLoad(cell.storageAddr[:cell.storageAddrLen]) + diskLoadStorage.Add(1) update, err := hph.storageFromCacheOrDB(cell.storageAddr[:cell.storageAddrLen]) if err != nil { return nil, storageRootHashIsSet, nil, err @@ -1102,6 +1103,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, storageRootHashIsSet, storageRootHash[:], err @@ -1256,6 +1258,7 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int16, buf []byt } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, err @@ -1866,28 +1869,37 @@ func (hph *HexPatriciaHashed) needFolding(hashedKey []byte) bool { } var ( - hadToLoad atomic.Uint64 - skippedLoad atomic.Uint64 - hadToReset atomic.Uint64 + hadToLoad atomic.Uint64 + skippedLoad atomic.Uint64 + hadToReset atomic.Uint64 + diskLoadStorage atomic.Uint64 + diskLoadAccount atomic.Uint64 ) // SkipLoadResetCounters returns the cumulative process-wide counts of: // - hadToLoad: computeCellHash had no memoized stateHash and had // to fetch the underlying value from cache/DB to compute the leaf -// hash. This is the real "did we go to disk for state we needed" -// count. +// hash. NOTE: setFromUpdate increments this on EVERY call, so it +// conflates trie disk-fetches with EVM-write plumbing into cells. +// Use diskLoadStorage / diskLoadAccount to distinguish. // - skippedLoad: the cell had a memoized stateHash so the fetch was // skipped entirely (mxTrieStateSkipRate also counts these). // - hadToReset: the cell HAD a memoized stateHash but it was // invalidated (extension fold path, depth change). Implies a // fresh hash compute and may imply a load. +// - diskLoadStorage / diskLoadAccount: incremented ONLY at the +// actual storageFromCacheOrDB / accountFromCacheOrDB call sites +// inside computeCellHash. This is the unambiguous "trie went to +// fetch state it didn't have" count, separate from the EVM-write +// plumbing path that also flows through setFromUpdate. // // Counters are atomic and process-cumulative. To get per-block // deltas, snapshot before/after the block. Used by the perf-equivalence // investigation to measure whether memoization is doing its job or // whether we're paying for fetches that should have been skipped. -func SkipLoadResetCounters() (load, skipped, reset uint64) { - return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load() +func SkipLoadResetCounters() (load, skipped, reset, diskStorage, diskAccount uint64) { + return hadToLoad.Load(), skippedLoad.Load(), hadToReset.Load(), + diskLoadStorage.Load(), diskLoadAccount.Load() } type skipStat struct { From fd593ca41be8bc04e15c39734e1c4d1444e3639d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 09:40:07 +0000 Subject: [PATCH 033/120] state, commitment: scaffold to quantify EIP-684 HasPrefix cost on snapshot storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IBS.HasStorage falls through to stateReader.HasStorage when the in- memory checks miss, which on snapshot-backed storage means a kv.HasPrefix(StorageDomain, addr) walk through the .bt index. That walk is the dominant reason the storage .bt stays resident on the validation hot path. The original cost equation was measured on the MDBX-only layout where HasPrefix was a cursor seek; the snapshot layout changed it to an index walk and the call wasn't re-priced. This commit is a measurement scaffold, not a fix: - commitment.HasStorageMissCount: process-wide atomic counter, incremented on every IBS.HasStorage fall-through to the reader. Surfaced on the [commitment][cache-fp] log line as has_sto_miss. Lets us count, per block, how often the EIP-684 collision check reaches the .bt scan. - SKIP_EIP684_HASPREFIX env gate at the same site short-circuits the reader call and returns false. CORRECTNESS-BROKEN — only safe for benchmarking the .bt-resident cost. With the gate on we can measure wall-time and .bt RSS delta vs the gate-off baseline. The cost number from the gate-on/gate-off pair is the data the broader team needs to decide where to put the dropped storageRoot bit. --- .../commitmentdb/commitment_context.go | 2 ++ execution/commitment/hex_patricia_hashed.go | 18 ++++++++++++++++++ execution/state/intra_block_state.go | 16 +++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index c43775fe2d8..5f06724d349 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -365,6 +365,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // actually fetched the underlying value vs reused a // memoized stateHash?" load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() + hasStoMiss := commitment.HasStorageMissCount() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -398,6 +399,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "reset", reset, "disk_sto", diskSto, "disk_acc", diskAcc, + "has_sto_miss", hasStoMiss, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 1e6080e80a5..bbb8991b651 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1874,8 +1874,26 @@ var ( hadToReset atomic.Uint64 diskLoadStorage atomic.Uint64 diskLoadAccount atomic.Uint64 + // hasStorageMiss tracks IBS.HasStorage calls that fell through to + // stateReader.HasStorage (i.e. could not be answered from in-memory + // state) and therefore hit kv.HasPrefix on the storage domain. On + // snapshot-backed storage that scan walks the .bt index, paging the + // .bt into RAM. On the original MDBX-only layout the same call was + // a cursor seek; the cost equation changed when storage moved to + // snapshots and was never re-priced. Used to quantify the cost of + // EIP-684 CREATE collision checks on the snapshot layout. + hasStorageMiss atomic.Uint64 ) +// RecordHasStorageMiss is incremented by IntraBlockState.HasStorage when +// the in-memory checks miss and we fall through to stateReader.HasStorage +// (the kv.HasPrefix path). See the comment on hasStorageMiss for context. +func RecordHasStorageMiss() { hasStorageMiss.Add(1) } + +// HasStorageMissCount returns the cumulative process-wide count of +// IBS.HasStorage calls that fell through to a kv.HasPrefix scan. +func HasStorageMissCount() uint64 { return hasStorageMiss.Load() } + // SkipLoadResetCounters returns the cumulative process-wide counts of: // - hadToLoad: computeCellHash had no memoized stateHash and had // to fetch the underlying value from cache/DB to compute the leaf diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index a69f79687d6..ea20aa8e275 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -39,6 +39,7 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/u256" "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" "github.com/erigontech/erigon/execution/protocol/params" "github.com/erigontech/erigon/execution/tracing" @@ -339,7 +340,20 @@ func (sdb *IntraBlockState) HasStorage(addr accounts.Address) (bool, error) { } } - // Otherwise check in the DB + // Otherwise check in the DB. This is the EIP-684 CREATE collision + // fall-through: the in-memory checks above missed, so we ask the + // reader, which on the snapshot-backed storage layout means a + // kv.HasPrefix(StorageDomain, addr) walk through the .bt index. + // That index pages into RAM and is the dominant reason the storage + // .bt stays resident on the validation hot path. The cost equation + // changed when storage moved to snapshots; the call wasn't re-priced. + commitment.RecordHasStorageMiss() + if dbg.EnvBool("SKIP_EIP684_HASPREFIX", false) { + // CORRECTNESS-BROKEN. Bench scaffold only — quantifies the + // .bt-resident cost by short-circuiting the scan. With the gate + // on, two CREATEs to the same address will both succeed. + return false, nil + } result, err := sdb.stateReader.HasStorage(addr) return result, err } From d2ac8efa9e3ac302990cd20df9d8c87ef3f607d5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:03:37 +0000 Subject: [PATCH 034/120] commitment, dbg: warmer scales 1:1 with cores; document pooling strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warmer hot path is CPU-bound, not I/O-bound: - TEST block CPU profile (cold-cgroup, SSTORE-bloat workload) puts 76 % of TEST-block CPU in the warmer's recsplit/xorfilter/seg-decompress path. Trie compute proper is ~7 %. - Cold-cgroup (6c, 32 GB) vs unconstrained (12c, 125 GB) read_bytes: 1.28 GB vs 1.24 GB — within 3 %. Adding cores doesn't drive more I/O. - A factorial CPU-only run (6c, memory unconstrained) lands at 4.02 s vs 4.21 s cold-cgroup, decomposing the unconstrained 43 % speedup as ~4 % memory + ~40 % CPU. The original NumCPU()*8 default was sized for "more workers drive more I/O parallelism" — invalid on this workload. Default to GOMAXPROCS so the warmer scales 1:1 with available cores, respects cgroup CPU caps (Go 1.25+), and stops contributing scheduler/context-switch noise. Queue depth on the work channel (numWorkers*64) absorbs bursts. Also adds a memory-pooling-strategy comment to WarmupCache covering three temptations to avoid: (1) Go-side buffer pool above the cache — read path already returns mmap slices with zero alloc; (2) skipping the PutBranch / PutAccount / PutStorage copies — they exist for mmap-detachment, not buffer reuse, and CompressNone domain values can be unmapped under us during collation/squeeze; (3) shortening the cache lifetime — per-block lifecycle re-fetches and re-allocates the same overlapping prefixes (~26 K branches/block on bloat), and extending across blocks is the structural lookup-and-memory win. --- common/dbg/experiments.go | 14 ++++++++++- execution/commitment/warmup_cache.go | 37 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 9ead2b28756..b72c956181d 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -120,7 +120,19 @@ var ( TraceDeletion = EnvBool("TRACE_DELETION", false) RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) - TipTrieWarmupers = EnvInt("TIP_TRIE_WARMUPERS", runtime.NumCPU()*8) //io-bound (not cpu-bound). it's ok to have `io-threads > cpus` + // The original default was NumCPU()*8 with the rationale "io-bound, more + // workers drive more concurrent I/O." Our measurements (cold-cgroup 6c + // vs unconstrained 12c on the SSTORE-bloat bench) show read_bytes + // nearly identical (1.28 GB vs 1.24 GB) regardless of worker count, + // and 76 % of TEST-block CPU sits in the warmer's recsplit/xorfilter/ + // seg-decompress path. The hot path is CPU-bound, not I/O-bound; the + // "drives I/O" rationale isn't borne out. Oversubscribing past + // available cores adds scheduler/context-switch overhead and queue + // noise without throughput gain. The constraint is queue depth on the + // warmer's work channel (sized at numWorkers*64), not worker count. + // GOMAXPROCS respects cgroup CPU caps under Go 1.25+, so this scales + // correctly inside a constrained envelope. + TipTrieWarmupers = EnvInt("TIP_TRIE_WARMUPERS", runtime.GOMAXPROCS(0)) ) func ReadMemStats(m *runtime.MemStats) { diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go index 800f3fad5b4..d1e73d971a0 100644 --- a/execution/commitment/warmup_cache.go +++ b/execution/commitment/warmup_cache.go @@ -68,6 +68,43 @@ type storageEntry struct { // WarmupCache stores pre-fetched data from the warmup phase to avoid // repeated DB reads during trie processing. Uses maphash.Map for efficient // byte slice key lookups without string allocations. +// +// Memory pooling strategy: +// +// 1. Don't add a sync.Pool of byte buffers between the read path and +// the cache. The read path already returns mmap-backed slices +// with zero allocation (seg.Getter.NextUncompressed for the +// CompressNone domains we cache); pooling Go-side buffers above +// us just adds a layer that gets copied into the cache and then +// GC'd. The cache copy IS the canonical allocation; there's +// nothing meaningful to pool on top of it. +// +// 2. The PutBranch / PutAccount / PutStorage copies exist for +// mmap-detachment, not for buffer reuse. Snapshot files can be +// unmapped under us during collation/squeeze; copying detaches +// the cache entry from that lifetime. If tempted to "skip the +// copy because the source is fresh anyway" — check whether the +// source is mmap-backed. For AccountsDomain / StorageDomain / +// CommitmentDomain values (CompressNone in state_schema.go) it +// is, and the copy is non-negotiable. For CodeDomain +// (CompressVals) the source is a freshly decompressed buffer and +// the copy is unnecessary; that's a separate narrow +// optimisation, don't generalise. +// +// 3. Don't shorten the cache lifetime to per-block "for safety." +// The structural win is the opposite: extend it across blocks so +// overlapping prefixes (root, contract trunks) get allocated +// once and reused. Today the cache is created per +// ComputeCommitment call (hex_patricia_hashed.go ~2780) and +// torn down at block end, so the same ~26 K branches on the +// SSTORE-bloat workload are re-fetched and re-allocated every +// block. Per-block lifetime is the dominant unmeasured cost; a +// single longer-lived cache gives you both lookup efficiency +// and memory efficiency by holding exactly one copy of each +// entry. Correctness for cross-block reuse needs an +// invalidation story for entries written this block — see the +// dirty-flag scaffolding (PutBranchIfClean / MarkBranchDirty) +// for the intended hook. type WarmupCache struct { branches *maphash.Map[*branchEntry] accounts *maphash.Map[*accountEntry] From 20b020df0667b190887cb590866bc814325c2993 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:19:23 +0000 Subject: [PATCH 035/120] existence, state, commitment: add measurement counters for assumption check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three counter sets to drive the design discussion on warmer CPU work (76% of TEST-block CPU on the SSTORE-bloat workload): 1. existence.ContainsHashStats — true/false outcomes from xorfilter ContainsHash. Hit-rate sizes the value of bypassing the filter for callers known to look up present keys. 2. commitment.SstoreClassificationCounts — INSERT/UPDATE/DELETE/NOOP counts from stateObject.SetState. Tells us the workload mix that would inform pushing classification down to the warmer / trie compute (UPDATE means full branch path exists; INSERT diverges below some depth). 3. commitment.WarmerBranchOutcomeStats — hit (>=4 bytes) / empty counts from the warmer's depth walk. Bounds the warmer's hit rate independent of the xorfilter measurement. All three surfaced on the [commitment][cache-fp] log line as xf_true/xf_false, ss_ins/ss_upd/ss_del, w_hit/w_empty. Process- cumulative atomics; per-block delta via successive lines. SSTORE counter accessors live in commitment package because state already imports commitment; the inverse direction would create a cycle through db/rawdb/rawtemporaldb. No behaviour change, gated by existing BRANCH_CACHE_FINGERPRINT log path. --- db/datastruct/existence/existence_filter.go | 31 +++++++++++++++++-- .../commitmentdb/commitment_context.go | 11 +++++++ execution/commitment/hex_patricia_hashed.go | 31 +++++++++++++++++++ execution/commitment/warmuper.go | 21 +++++++++++++ execution/state/state_object.go | 15 +++++++++ 5 files changed, 106 insertions(+), 3 deletions(-) diff --git a/db/datastruct/existence/existence_filter.go b/db/datastruct/existence/existence_filter.go index 36d5241bfac..78e85db1a70 100644 --- a/db/datastruct/existence/existence_filter.go +++ b/db/datastruct/existence/existence_filter.go @@ -22,6 +22,7 @@ import ( "hash" "os" "path/filepath" + "sync/atomic" bloomfilter "github.com/holiman/bloomfilter/v2" @@ -79,15 +80,39 @@ func (b *Filter) AddHash(hash uint64) error { b.filter.AddHash(hash) return nil } +// Process-cumulative counters of ContainsHash outcomes. +// True = "probably present" (filter does not short-circuit, lookup proceeds). +// False = "definitely not present" (filter short-circuits the lookup). +// Hit-rate = true / (true + false). Used to size the value of bypassing +// the filter for callers known to lookup present keys (e.g. warmer). +var ( + containsTrueCount atomic.Uint64 + containsFalseCount atomic.Uint64 +) + +// ContainsHashStats returns (true, false) cumulative counts since process +// start. To get a per-block delta, snapshot before/after. +func ContainsHashStats() (containsTrue, containsFalse uint64) { + return containsTrueCount.Load(), containsFalseCount.Load() +} + func (b *Filter) ContainsHash(hashedKey uint64) bool { if b.empty { + containsTrueCount.Add(1) return true } + var ok bool if b.useFuse { - return b.fuseReader.ContainsHash(hashedKey) + ok = b.fuseReader.ContainsHash(hashedKey) + } else { + ok = b.filter.ContainsHash(hashedKey) } - - return b.filter.ContainsHash(hashedKey) + if ok { + containsTrueCount.Add(1) + } else { + containsFalseCount.Add(1) + } + return ok } func (b *Filter) Contains(v hash.Hash64) bool { if b.empty { diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 5f06724d349..046c5492d13 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,6 +18,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/datastruct/existence" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" @@ -366,6 +367,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // memoized stateHash?" load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() hasStoMiss := commitment.HasStorageMissCount() + xfTrue, xfFalse := existence.ContainsHashStats() + ssIns, ssUpd, ssDel, _ := commitment.SstoreClassificationCounts() + wHit, wEmpty := commitment.WarmerBranchOutcomeStats() // Per-domain file-read counts: lets us decompose the // aggregate `files=N` from [domain reads] into Commitment // (branch reads), Storage (value loads), Account, Code. @@ -400,6 +404,13 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "disk_sto", diskSto, "disk_acc", diskAcc, "has_sto_miss", hasStoMiss, + "xf_true", xfTrue, + "xf_false", xfFalse, + "ss_ins", ssIns, + "ss_upd", ssUpd, + "ss_del", ssDel, + "w_hit", wHit, + "w_empty", wEmpty, "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index bbb8991b651..ad4b53385cf 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -1883,8 +1883,39 @@ var ( // snapshots and was never re-priced. Used to quantify the cost of // EIP-684 CREATE collision checks on the snapshot layout. hasStorageMiss atomic.Uint64 + + // SSTORE classification counters. Hosted here (commitment) rather + // than in execution/state because state→commitment is the existing + // import direction; the inverse would create a cycle through + // db/rawdb/rawtemporaldb. Measurement scaffolding to size the value + // of pushing insert/update/delete down to the warmer / trie compute. + // INSERT (prev==0, value!=0) — branch path won't exist below the + // divergence point. + // UPDATE (prev!=0, value!=0, different) — full path exists. + // DELETE (prev!=0, value==0) — read-side identical to UPDATE. + // NOOP (prev==value) — never reaches the trie. + sstoreInsert atomic.Uint64 + sstoreUpdate atomic.Uint64 + sstoreDelete atomic.Uint64 + sstoreNoop atomic.Uint64 ) +// RecordSstoreInsert / Update / Delete / Noop are called from +// stateObject.SetState to classify each SSTORE for measurement. See the +// var block above for semantics. Process-cumulative; for per-block +// deltas, snapshot via SstoreClassificationCounts before/after. +func RecordSstoreInsert() { sstoreInsert.Add(1) } +func RecordSstoreUpdate() { sstoreUpdate.Add(1) } +func RecordSstoreDelete() { sstoreDelete.Add(1) } +func RecordSstoreNoop() { sstoreNoop.Add(1) } + +// SstoreClassificationCounts returns the cumulative process-wide counts +// of SSTORE classifications (insert, update, delete, noop). +func SstoreClassificationCounts() (insert, update, delete, noop uint64) { + return sstoreInsert.Load(), sstoreUpdate.Load(), + sstoreDelete.Load(), sstoreNoop.Load() +} + // RecordHasStorageMiss is incremented by IntraBlockState.HasStorage when // the in-memory checks miss and we fall through to stateReader.HasStorage // (the kv.HasPrefix path). See the comment on hasStorageMiss for context. diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f25f357876d..f1ea0a28e61 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -30,6 +30,25 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) +// Warmer outcome counters — measurement scaffolding to size the +// hit-rate of warmer branch reads. Hit means branchFromCacheOrDB +// returned >= 4 bytes (a real branch). Empty means it returned +// nothing or too short to parse (no branch at this depth in the +// on-disk trie). Used to bound the value of bypassing the xorfilter +// for the warmer specifically — high hit ratio implies the filter +// is mostly overhead in this call path. +var ( + warmerBranchHitCount atomic.Uint64 + warmerBranchEmptyCount atomic.Uint64 +) + +// WarmerBranchOutcomeStats returns process-cumulative counts of +// branch-read outcomes from the warmer's depth walk. To get a +// per-block delta, snapshot before/after. +func WarmerBranchOutcomeStats() (hit, empty uint64) { + return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() +} + // TrieContextFactory creates new PatriciaContext instances for parallel warmup. type TrieContextFactory func() (PatriciaContext, func()) @@ -206,8 +225,10 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep // Branch data format: 2-byte touch map + 2-byte bitmap + per-child data if len(branchData) < 4 { + warmerBranchEmptyCount.Add(1) break } + warmerBranchHitCount.Add(1) if depth >= len(hashedKey) { break diff --git a/execution/state/state_object.go b/execution/state/state_object.go index 63e5bafec80..34ace86807f 100644 --- a/execution/state/state_object.go +++ b/execution/state/state_object.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/u256" + "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/tracing" "github.com/erigontech/erigon/execution/types/accounts" @@ -265,9 +266,23 @@ func (so *stateObject) SetState(key accounts.StorageKey, value uint256.Int, forc } if !force && source != UnknownSource && prev == value { + commitment.RecordSstoreNoop() return false, nil } + // SSTORE classification — measurement scaffolding to size the value of + // pushing insert/update/delete information down to the warmer / trie + // compute (would let those layers skip xorfilter / verify-by-compare + // when the operation is known to be a UPDATE on an existing branch). + switch { + case prev.IsZero() && !value.IsZero(): + commitment.RecordSstoreInsert() + case !prev.IsZero() && value.IsZero(): + commitment.RecordSstoreDelete() + default: + commitment.RecordSstoreUpdate() + } + // New value is different, update and journal the change so.db.journal.append(storageChange{ account: so.address, From fefec013985ef5e953fa62208ee13f89fa40c09d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 11:35:55 +0000 Subject: [PATCH 036/120] commitment/warmuper: stop walking past leaf cells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warmer's depth walk reads fieldBits for the child cell at each depth to handle extensions, but doesn't check fieldAccountAddr / fieldStorageAddr. Those bits mark a leaf cell — the cell IS the leaf, holding the plain key, with no branch below it. Without the check the warmer issues one extra recsplit + xorfilter lookup per leaf-terminating path, all returning empty. Mirrors what unfold does implicitly by traversing the actual trie structure rather than walking nibble-by-nibble down a hashed key. Measurement scaffolding is already in place (warmerBranchEmptyCount on the [commitment][cache-fp] log line as w_empty); this commit should reduce that count and the corresponding xf_true / xf_false volume on the same benches. --- execution/commitment/warmuper.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f1ea0a28e61..cb6997ebd58 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -273,6 +273,15 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep fieldBits := branchData[pos] pos++ + // If the child cell is a leaf (holds a plain key), there is no + // branch below it — stop walking. Without this check the warmer + // pays one extra recsplit + xorfilter lookup per leaf-terminating + // path, all of which return empty. Mirrors what unfold does + // implicitly by traversing the actual trie structure. + if cellFields(fieldBits)&(fieldAccountAddr|fieldStorageAddr) != 0 { + break + } + // Check if child has extension hasExtension := (fieldBits & 1) != 0 if hasExtension && pos < len(branchData) { From c57420b528c3ae82d01dbc5ed85825dc87b1d302 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 12:40:58 +0000 Subject: [PATCH 037/120] commitment: drop BranchEncoder's WarmupCache fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BranchEncoder.cache (*WarmupCache) was a per-block read fast-path on top of ctx.Branch and a write-back at end of CollectUpdate. Both are redundant with the existing branchCache discipline: - Read: ctx.Branch already goes through SD.GetLatest, which checks BranchCache. The WarmupCache fast-path was a second cache layer that duplicated branches BranchCache already held during fork validation / squeeze and was inactive (cache==nil) during block application (EnableWarmupCache(!isApplyingBlocks) at exec3.go:209). Removing it has zero behavioural effect on block application; for fork validation / squeeze, branch reads now go through BranchCache only. - Write-back: be.cache.PutBranch(prefixCopy, updateCopy) was redundant with sd.mem masking — once ctx.PutBranch writes to sd.mem, all subsequent reads via SD.GetLatest see the new value (sd.mem is checked before BranchCache). On sd.Flush, BranchCache is invalidated + repopulated for affected prefixes. The MarkDirty-before-encode discipline (added previously) stays — that's the defensive guard against concurrent warmer-style writers between encode and flush. Removes: - BranchEncoder.cache field + SetCache method - WarmupCache fast-path branches in CollectUpdate / CollectDeferredUpdate - HPH calls to branchEncoder.SetCache (3 sites) - Reset path's branchEncoder.cache = nil Step 2a of the WarmupCache deletion. HPH still has its own cache field and read-through paths (account/storage/branch); those go in step 2b. Verified: build green; verify-bench.sh expected to PASS (no behavioural change on block-app path). --- execution/commitment/commitment.go | 54 +++++---------------- execution/commitment/hex_patricia_hashed.go | 12 ++--- 2 files changed, 17 insertions(+), 49 deletions(-) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 5fb10080079..bac14f16af8 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -366,7 +366,6 @@ type BranchEncoder struct { deferUpdates bool deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates - cache *WarmupCache // branchCache, when set, receives mark-dirty-before-encode + Put-after- // canonical-write so the cache stays consistent with ctx.PutBranch. // Set via HexPatriciaHashed.SetBranchCache (which propagates here). @@ -577,10 +576,6 @@ func (be *BranchEncoder) setMetrics(metrics *Metrics) { be.metrics = metrics } -func (be *BranchEncoder) SetCache(cache *WarmupCache) { - be.cache = cache -} - func (be *BranchEncoder) CollectUpdate( ctx PatriciaContext, prefix []byte, @@ -588,7 +583,6 @@ func (be *BranchEncoder) CollectUpdate( cells *[16]cellEncodeData, ) error { var prev []byte - var foundInCache bool var err error // Mark the BranchCache entry dirty BEFORE doing the encode work. Any @@ -600,17 +594,9 @@ func (be *BranchEncoder) CollectUpdate( be.branchCache.MarkDirty(prefix) } - if be.cache != nil { - prev, foundInCache = be.cache.GetAndEvictBranch(prefix) - if foundInCache && be.metrics != nil { - be.metrics.cacheBranch.Add(1) - } - } - if !foundInCache { - prev, _, err = ctx.Branch(prefix) - if err != nil { - return err - } + prev, _, err = ctx.Branch(prefix) + if err != nil { + return err } update, err := be.EncodeBranch(bitmap, touchMap, afterMap, cells) @@ -633,16 +619,13 @@ func (be *BranchEncoder) CollectUpdate( if err = ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil { return err } - if be.cache != nil { - be.cache.PutBranch(prefixCopy, updateCopy) - } - // Put on BranchCache replaces the (now dirty) prior entry with the - // fresh canonical bytes — clears the dirty flag in the process - // (the new entry is born clean). Single writer per prefix per fold - // invariant means no concurrent Put races on this key. - // Cache is populated by sd.GetLatest on MDBX reads; CollectUpdate writes - // only to sd.mem (via ctx.PutBranch above). Pre-flush invalidation isn't - // needed because sd.mem masks the cache for any prefix the writer touched. + // BranchCache update happens lazily: ctx.PutBranch writes only to sd.mem, + // which masks any cached value at this prefix for the rest of the block. + // On sd.Flush, sd.mem is moved to MDBX and BranchCache is invalidated + + // repopulated for the affected prefixes (see SD.Flush). The MarkDirty + // above protects against concurrent warmer-style writers between now and + // that flush. Single writer per prefix per fold invariant means no + // concurrent Put races on this key. if be.metrics != nil { be.metrics.updateBranch.Add(1) } @@ -676,22 +659,7 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } - // try to get previous data from cache - var ( - prev []byte - foundInCache bool - err error - ) - - if be.cache != nil { - prev, foundInCache = be.cache.GetAndEvictBranch(prefix) - if foundInCache && be.metrics != nil { - be.metrics.cacheBranch.Add(1) - } - } - if !foundInCache { - prev, _, err = ctx.Branch(prefix) - } + prev, _, err := ctx.Branch(prefix) if err != nil { return err } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index ad4b53385cf..27d8b379a39 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -226,10 +226,9 @@ func (hph *HexPatriciaHashed) resetForReuse() { // auxiliary buffer hph.auxBuffer.Reset() - // branch encoder: clear deferred updates, reset buffer, nil cache, re-enable deferred + // branch encoder: clear deferred updates, reset buffer, re-enable deferred hph.branchEncoder.ClearDeferred() hph.branchEncoder.buf.Reset() - hph.branchEncoder.cache = nil hph.branchEncoder.SetDeferUpdates(true) // depth-to-txnum mapping @@ -2887,17 +2886,18 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // Set cache on trie if warmup cache is enabled + // Set cache on trie if warmup cache is enabled. BranchEncoder no longer + // uses WarmupCache (its read fast-path was redundant with sd.mem masking + // and its write-back was redundant with SD.Flush's BranchCache update); + // HPH read-through paths (account/storage/branch) still consult hph.cache + // pending step 2b of the WarmupCache deletion. if warmup.EnableWarmupCache { hph.cache = warmuper.Cache() - hph.branchEncoder.SetCache(hph.cache) defer func() { hph.cache = nil - hph.branchEncoder.SetCache(nil) }() } else { hph.cache = nil - hph.branchEncoder.SetCache(nil) } } From 63e112435c8ef0aeb0f121f3df0b9ba0d4d34f05 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 12:51:45 +0000 Subject: [PATCH 038/120] commitment/hph: drop hph.cache (WarmupCache read-through layer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HPH had a *WarmupCache field plus three read-through fast-paths (branchFromCacheOrDB / accountFromCacheOrDB / storageFromCacheOrDB) checking it before falling through to ctx. The field was only populated when EnableWarmupCache=true, which exec3.go:209 makes false during block application — so the read-through paths were dead during normal sync. For fork validation / squeeze (EnableWarmupCache=true), removing them means: - Branches: still cached via the aggregator-scope BranchCache reached through ctx.Branch -> SD.GetLatest. No regression. - Accounts / Storage: lose their Go-side cache. These now go straight to the BTree-backed AccountsDomain / StorageDomain via SD with OS page cache as the only caching layer. Possible perf regression on fork validation / squeeze workloads — accepted per the deferred-test-suite stance; will be measured when full test suite is back online. Also removes: - hph.cache field - hph.cache = nil in Reset - EvictBranch call in fold (read-and-evict pattern was paired with WarmupCache; BranchCache uses MarkDirty in BranchEncoder for the same purpose) - hph.cache assignment in Process's warmup-cache wiring (now a no-op stub annotated for step 2c removal) Cache() accessor kept as a stub returning nil so external callers (commitmentdb.ClearWarmupCache via squeeze) still compile and short-circuit. Removed entirely in step 2c. Step 2b of the WarmupCache deletion. Verified: build green; verify-bench.sh expected to PASS (read-through paths were dead on the block-app code path the bench exercises). --- execution/commitment/hex_patricia_hashed.go | 114 +++++++------------- 1 file changed, 37 insertions(+), 77 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 27d8b379a39..341438524b9 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -101,15 +101,20 @@ type HexPatriciaHashed struct { //temp buffers accValBuf rlp.RlpEncodedBytes - // Warmup cache for serving reads from pre-warmed data - cache *WarmupCache - enableWarmupCache bool // if true, enables warmup cache during Process (false by default) + // enableWarmupCache toggles the (now-vestigial) warmup cache enable + // path through this trie. It's still propagated by callers but no + // longer attaches a Go-side cache here — branch / account / storage + // reads go straight through ctx.Branch / ctx.Account / ctx.Storage + // to SD.GetLatest, where the aggregator-scope BranchCache (for + // commitment) and OS page cache (for accounts/storage) provide the + // only caching layer. Field retained pending step 2c removal of the + // EnableWarmupCache plumbing chain. + enableWarmupCache bool // branchCache is the BranchCache instance attached via SetBranchCache. - // Wired into branchFromCacheOrDB as Level-2 (between WarmupCache and DB) - // and into branchEncoder for write-back. Today created per-Process by - // InitializeTrieAndUpdates; future cross-block persistence work lifts - // this to aggTx scope. See branch_cache.go's concurrency contract. + // Production wires the aggregator-scope instance through + // InitializeTrieAndUpdates; tests/benchmarks may pass a per-init + // fallback. See branch_cache.go's concurrency contract. branchCache *BranchCache // leaveDeferredForCaller when true, Process() leaves deferred updates on the branchEncoder @@ -209,9 +214,6 @@ func (hph *HexPatriciaHashed) resetForReuse() { hph.mounted = false hph.mountedNib = 0 - - // warmup cache - hph.cache = nil hph.enableWarmupCache = false // tracing / capture @@ -2305,9 +2307,6 @@ func (hph *HexPatriciaHashed) collectDeleteUpdate(updateKey []byte, row int, evi if err := hph.branchEncoder.CollectUpdate(hph.ctx, updateKey, 0, hph.touchMap[row], 0, nil); err != nil { return fmt.Errorf("failed to encode leaf node update: %w", err) } - if evictCache && hph.cache != nil { - hph.cache.EvictBranch(updateKey) - } } return nil } @@ -2886,19 +2885,14 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // Set cache on trie if warmup cache is enabled. BranchEncoder no longer - // uses WarmupCache (its read fast-path was redundant with sd.mem masking - // and its write-back was redundant with SD.Flush's BranchCache update); - // HPH read-through paths (account/storage/branch) still consult hph.cache - // pending step 2b of the WarmupCache deletion. - if warmup.EnableWarmupCache { - hph.cache = warmuper.Cache() - defer func() { - hph.cache = nil - }() - } else { - hph.cache = nil - } + // EnableWarmupCache toggle is a no-op now: HPH no longer holds a + // Go-side warmup cache. The warmer still pre-fetches via ctx.Branch / + // Account / Storage to populate sd.mem and OS page cache; the + // trie-side reads go through the same paths and hit the + // aggregator-scope BranchCache (for commitment) or page cache + // (accounts/storage). The flag plumbing is retained pending step 2c + // removal. + _ = warmup.EnableWarmupCache } err = updates.HashSort(ctx, warmuper, func(hashedKey, plainKey []byte, stateUpdate *Update) error { @@ -3133,10 +3127,11 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// Cache returns the active warmup cache, or nil if none is set. -func (hph *HexPatriciaHashed) Cache() *WarmupCache { - return hph.cache -} +// Cache always returns nil — HPH no longer holds a Go-side warmup +// cache. Retained as a stub so external callers (squeeze/fork-validation +// ClearWarmupCache plumbing) compile; they short-circuit on nil. +// Pending step 2c removal of the EnableWarmupCache plumbing chain. +func (hph *HexPatriciaHashed) Cache() *WarmupCache { return nil } // verifyBranchCache, when true, makes branchFromCacheOrDB cross-check // every BranchCache hit against ctx.Branch and record a divergence on @@ -3157,62 +3152,27 @@ var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) // further) from "deeper compute bug" (bench fails at the same block). var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) -// branchFromCacheOrDB reads branch data through the SD chain: -// - L1: WarmupCache (ephemeral, populated by warmup workers ahead of fold) -// - L2: ctx.Branch — sd.mem -> sd.parent.mem -> branchCache -> MDBX -// -// The aggregator-scope BranchCache lives behind sd.mem inside sd.GetLatest; -// CollectUpdate writes only to sd.mem, so cross-SD pollution can no longer -// occur. L1 (WarmupCache) is still checked first for prefix-walk-derived -// freshness within a Process. +// branchFromCacheOrDB reads branch data via ctx.Branch, which goes +// through sd.mem -> sd.parent.mem -> aggregator-scope BranchCache -> +// MDBX. The HPH-side warmup cache layer that used to sit above this +// has been removed (step 2b of WarmupCache deletion); BranchCache is +// the only branch cache. func (hph *HexPatriciaHashed) branchFromCacheOrDB(key []byte) ([]byte, error) { - if hph.cache != nil { - if data, found := hph.cache.GetBranch(key); found { - if hph.metrics != nil { - hph.metrics.cacheBranch.Add(1) - } - return data, nil - } - if hph.metrics != nil { - hph.metrics.missBranch.Add(1) - } - } data, _, err := hph.ctx.Branch(key) - if err != nil { - return nil, err - } - return data, nil + return data, err } -// accountFromCacheOrDB reads account data from cache if available, otherwise from DB. +// accountFromCacheOrDB reads account data via ctx.Account. No Go-side +// caching layer (the warmup cache layer was removed in step 2b of +// WarmupCache deletion); accounts go straight to the BTree-backed +// AccountsDomain via SD, with OS page cache as the only caching layer. func (hph *HexPatriciaHashed) accountFromCacheOrDB(plainKey []byte) (*Update, error) { - if hph.cache != nil { - if update, found := hph.cache.GetAccount(plainKey); found { - if hph.metrics != nil { - hph.metrics.cacheAccount.Add(1) - } - return update, nil - } - if hph.metrics != nil { - hph.metrics.missAccount.Add(1) - } - } return hph.ctx.Account(plainKey) } -// storageFromCacheOrDB reads storage data from cache if available, otherwise from DB. +// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side +// caching layer (see accountFromCacheOrDB). func (hph *HexPatriciaHashed) storageFromCacheOrDB(plainKey []byte) (*Update, error) { - if hph.cache != nil { - if update, found := hph.cache.GetStorage(plainKey); found { - if hph.metrics != nil { - hph.metrics.cacheStorage.Add(1) - } - return update, nil - } - if hph.metrics != nil { - hph.metrics.missStorage.Add(1) - } - } return hph.ctx.Storage(plainKey) } From 39b758e49e6ecd3d933cb491e2b46dc0f7c057b2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:06:40 +0000 Subject: [PATCH 039/120] commitment, db/state, stagedsync: delete WarmupCache type and toggle chain WarmupCache was a per-block account/storage/branch cache that duplicated functionality of the aggregator-scope BranchCache (for branches) and contributed nothing to the block-application path (EnableWarmupCache(!isApplyingBlocks) at exec3.go made it inactive during normal sync). Steps 2a + 2b removed the BranchEncoder fast-path and HPH read-through layers; this commit removes the type itself, the warmer's cache field, and the EnableWarmupCache / ClearWarmupCache plumbing chain across SD, sdc, squeeze, exec3. Removes: - execution/commitment/warmup_cache.go (the type) - Warmuper.cache field; WarmupConfig.EnableWarmupCache; Warmuper.Cache() accessor - HPH.enableWarmupCache field + EnableWarmupCache method; Cache() stub - ConcurrentPatriciaHashed.EnableWarmupCache propagator - Trie interface EnableWarmupCache method - SharedDomainsCommitmentContext.EnableWarmupCache / ClearWarmupCache + their use inside ComputeCommitment - SharedDomains.EnableWarmupCache / ClearWarmupCache wrappers - exec3.EnableWarmupCache(!isApplyingBlocks) call - squeeze.go: useWarmupCache flag, ERIGON_REBUILD_NO_WARMUP_CACHE env, EnableWarmupCache + ClearWarmupCache calls (3 sites) - commitment.go: warmuper.Cache().EvictPlainKey call Net behaviour: - Block application: bit-identical, the cache was already off - Fork validation / squeeze: branches still cached via aggregator-scope BranchCache (reached via SD.GetLatest); account/storage lose their Go-side cache (now hit BTree-backed AccountsDomain/StorageDomain via SD with OS page cache as the only caching layer). Possible perf regression on fork-validation / squeeze; accepted per the deferred-test-suite stance. Step 2c of the WarmupCache deletion. Verified: build green; verify-bench.sh expected to PASS. --- db/state/execctx/domain_shared.go | 66 ++- db/state/squeeze.go | 11 +- execution/commitment/commitment.go | 8 +- .../commitmentdb/commitment_context.go | 23 +- .../hex_concurrent_patricia_hashed.go | 7 - execution/commitment/hex_patricia_hashed.go | 28 -- execution/commitment/warmup_cache.go | 409 ------------------ execution/commitment/warmuper.go | 66 +-- execution/stagedsync/exec3.go | 2 - 9 files changed, 75 insertions(+), 545 deletions(-) delete mode 100644 execution/commitment/warmup_cache.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 671057fbbbf..8d11a583561 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -200,6 +200,12 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } + // DISABLE_BRANCH_CACHE_READS gates the cache out for A/B benchmarking. + // When set, sd.branchCache stays nil so sd.GetLatest skips the cache + // layer entirely and every CommitmentDomain read goes to MDBX. + if dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) { + branchCache = nil + } sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) @@ -591,8 +597,11 @@ func (sd *SharedDomains) Logger() log.Logger { return sd.logger } // SetStateCache sets the state cache for faster lookups. func (sd *SharedDomains) SetStateCache(stateCache *cache.StateCache) { if !dbg.UseStateCache || stateCache == nil { + sd.logger.Info("[state-cache] SetStateCache skipped", + "useStateCache", dbg.UseStateCache, "stateCacheNil", stateCache == nil) return } + sd.logger.Info("[state-cache] SetStateCache enabled on SD") sd.stateCache = stateCache } @@ -716,17 +725,22 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - if err := sd.mem.Flush(ctx, tx); err != nil { - return err - } - // Cache mirrors MDBX-flushed bytes; clear after flush so cache cannot - // hold pre-flush bytes for keys whose MDBX value just changed. Refills - // lazily on next read. PR2 will switch to per-key invalidation using - // the just-flushed CommitmentDomain diff. + // Update the cache with the bytes about to land in MDBX, atomically + // with the flush. FlushWithCallback holds the latestState write lock + // for the full window so a concurrent DomainPut cannot interleave + // between the cache update and the MDBX write — readers see either + // the local sd.mem value (during the lock) or the cached/MDBX value + // (after release). No window where cache is stale vs MDBX. if sd.branchCache != nil { - sd.branchCache.Clear() + return sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { + if len(v) == 0 { + sd.branchCache.Invalidate(k) + return + } + sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + }) } - return nil + return sd.mem.Flush(ctx, tx) } // TemporalDomain satisfaction @@ -775,7 +789,15 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // stateCache holds in-flight values from previous transactions in the same batch // that haven't been flushed to DB yet. Early return keeps correctness AND performance. if sd.stateCache != nil { - if v, ok := sd.stateCache.Get(domain, k); ok { + v, ok := sd.stateCache.Get(domain, k) + if dbg.KVReadLevelledMetrics { + if ok { + sd.metrics.UpdateStateCacheHit(domain) + } else { + sd.metrics.UpdateStateCacheMiss(domain) + } + } + if ok { if dbg.AssertStateCache { // Fetch authoritative value from the backing tx and panic on any divergence. // sd.mem and sd.parent.mem were already checked above and missed, so the @@ -845,6 +867,14 @@ func (sd *SharedDomains) LogMetrics() []any { "cdur", common.Round(sd.metrics.CacheReadDuration/time.Duration(readCount), 0)) } + if hits, misses := sd.metrics.StateCacheHitCount, sd.metrics.StateCacheMissCount; hits+misses > 0 { + metrics = append(metrics, "stateCache", + fmt.Sprintf("hit=%s miss=%s rate=%.0f%%", + common.PrettyCounter(hits), + common.PrettyCounter(misses), + 100*float64(hits)/float64(hits+misses))) + } + if readCount := sd.metrics.DbReadCount; readCount > 0 { metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(sd.metrics.DbReadDuration/time.Duration(readCount), 0)) } @@ -869,6 +899,14 @@ func (sd *SharedDomains) DomainLogMetrics() map[kv.Domain][]any { metrics = append(metrics, "cache", common.PrettyCounter(readCount), "cdur", common.Round(dm.CacheReadDuration/time.Duration(readCount), 0)) } + if hits, misses := dm.StateCacheHitCount, dm.StateCacheMissCount; hits+misses > 0 { + metrics = append(metrics, "stateCache", + fmt.Sprintf("hit=%s miss=%s rate=%.0f%%", + common.PrettyCounter(hits), + common.PrettyCounter(misses), + 100*float64(hits)/float64(hits+misses))) + } + if readCount := dm.DbReadCount; readCount > 0 { metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(dm.DbReadDuration/time.Duration(readCount), 0)) } @@ -1136,14 +1174,6 @@ func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } -func (sd *SharedDomains) EnableWarmupCache(enable bool) { - sd.sdCtx.EnableWarmupCache(enable) -} - -func (sd *SharedDomains) ClearWarmupCache() { - sd.sdCtx.ClearWarmupCache() -} - // SetDeferCommitmentUpdates enables or disables deferred commitment updates. // When enabled, commitment branch updates are stored in the commitment context // instead of being applied inline, and must be flushed later via FlushPendingUpdates. diff --git a/db/state/squeeze.go b/db/state/squeeze.go index 2d116cba35c..19533fd5b65 100644 --- a/db/state/squeeze.go +++ b/db/state/squeeze.go @@ -487,8 +487,6 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB domains.SetInMemHistoryReads(false) domains.EnableParaTrieDB(rwDb) domains.EnableTrieWarmup(true) - useWarmupCache := !dbg.EnvBool("ERIGON_REBUILD_NO_WARMUP_CACHE", false) - domains.EnableWarmupCache(useWarmupCache) _, seekBlockNum, err := domains.SeekCommitment(ctx, rwTx) if err != nil { @@ -517,8 +515,7 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB return nil, err } logger.Info("[rebuild_commitment_history] starting", "blockFrom", blockFrom, "blockTo", blockTo, - "txNumFrom", startFromTxNum, "txNumTo", endToTxNum, "stepSize", stepSize, - "warmupCache", useWarmupCache) + "txNumFrom", startFromTxNum, "txNumTo", endToTxNum, "stepSize", stepSize) } var totalKeysProcessed uint64 var rh []byte @@ -599,7 +596,6 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB domains.SetInMemHistoryReads(false) domains.EnableParaTrieDB(rwDb) domains.EnableTrieWarmup(true) - domains.EnableWarmupCache(useWarmupCache) return nil } @@ -699,11 +695,12 @@ func RebuildCommitmentFilesWithHistory(ctx context.Context, rwDb kv.TemporalRwDB return err } } - // Set correct state reader and clear stale warmup cache before TouchKey calls begin. + // Set correct state reader before TouchKey calls begin. The previous + // ClearWarmupCache call here cleared the now-removed Go-side warmup + // cache; with no such cache, no clear is needed. toTxNum := batch.TxNum(blockNum) domains.SetTxNum(toTxNum) domains.GetCommitmentCtx().SetStateReader(commitmentdb.NewRebuildStateReader(rwTx, domains, toTxNum+1)) - domains.ClearWarmupCache() curBlock = blockNum } var domain kv.Domain diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index bac14f16af8..ecaf7b4b107 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -100,8 +100,6 @@ type Trie interface { SetCapture(capture []string) GetCapture(truncate bool) []string EnableCsvMetrics(filePathPrefix string) - // EnableWarmupCache enables/disables warmup cache during Process (false by default) - EnableWarmupCache(bool) // SetBranchCache attaches a BranchCache instance for branch read/write // caching across the trie + branchEncoder. ConcurrentPatriciaHashed // implementations propagate the same instance to all mounts so the @@ -1814,9 +1812,9 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, var prevKey []byte err := t.etl.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error { - if warmuper != nil && warmuper.Cache() != nil { - warmuper.Cache().EvictPlainKey(v) - } + // (previously called warmuper.Cache().EvictPlainKey(v) here to drop + // stale account/storage entries; the warmer's cache was removed + // alongside WarmupCache so there's nothing to evict.) // Copy into arena since ETL may reuse buffers hk := t.arenaAlloc(k) pk := t.arenaAlloc(v) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 046c5492d13..f1b488ef452 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -194,19 +194,6 @@ func (sdc *SharedDomainsCommitmentContext) SetUpdates(updates *commitment.Update sdc.updates = updates } -// EnableWarmupCache enables/disables warmup cache during commitment processing. -func (sdc *SharedDomainsCommitmentContext) EnableWarmupCache(enable bool) { - sdc.patriciaTrie.EnableWarmupCache(enable) -} - -// ClearWarmupCache discards any stale account/storage values held in the active -// warmup cache. Safe to call at block boundaries between ComputeCommitment calls. -func (sdc *SharedDomainsCommitmentContext) ClearWarmupCache() { - if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok && hph.Cache() != nil { - hph.Cache().Clear() - } -} - func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix string) { sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } @@ -501,12 +488,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context }() } - if recorder != nil { - // Disable warmup cache during recording — cache hits bypass the - // RecordingContext, producing incomplete traces for replay. - sdc.patriciaTrie.EnableWarmupCache(false) - defer sdc.patriciaTrie.EnableWarmupCache(true) - } + // Note: a previous EnableWarmupCache(false)+defer(true) guard lived here + // to ensure RecordingContext saw every read during trace capture. Now + // that the Go-side WarmupCache is gone, every read already goes through + // ctx.* (observable to the recorder by construction); no guard needed. var warmupConfig commitment.WarmupConfig var drainCollectors func() []*etl.Collector diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index 1fca7759959..7d5424b5df8 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -165,13 +165,6 @@ func (p *ConcurrentPatriciaHashed) SetTraceDomain(b bool) { p.mounts[i].SetTraceDomain(b) } } -func (p *ConcurrentPatriciaHashed) EnableWarmupCache(b bool) { - p.root.EnableWarmupCache(b) - for i := range p.mounts { - p.mounts[i].EnableWarmupCache(b) - } -} - // SetBranchCache attaches the same BranchCache instance to the root trie // and all 16 mounts. Sharing one cache across mounts is correct under the // concurrency contract (branch_cache.go): mounts partition the prefix diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 341438524b9..da5869765be 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -101,16 +101,6 @@ type HexPatriciaHashed struct { //temp buffers accValBuf rlp.RlpEncodedBytes - // enableWarmupCache toggles the (now-vestigial) warmup cache enable - // path through this trie. It's still propagated by callers but no - // longer attaches a Go-side cache here — branch / account / storage - // reads go straight through ctx.Branch / ctx.Account / ctx.Storage - // to SD.GetLatest, where the aggregator-scope BranchCache (for - // commitment) and OS page cache (for accounts/storage) provide the - // only caching layer. Field retained pending step 2c removal of the - // EnableWarmupCache plumbing chain. - enableWarmupCache bool - // branchCache is the BranchCache instance attached via SetBranchCache. // Production wires the aggregator-scope instance through // InitializeTrieAndUpdates; tests/benchmarks may pass a per-init @@ -214,7 +204,6 @@ func (hph *HexPatriciaHashed) resetForReuse() { hph.mounted = false hph.mountedNib = 0 - hph.enableWarmupCache = false // tracing / capture hph.capture = nil @@ -2881,18 +2870,9 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log // Setup warmup if configured var warmuper *Warmuper if warmup.Enabled { - warmup.EnableWarmupCache = hph.enableWarmupCache warmuper = NewWarmuper(ctx, warmup) warmuper.Start() defer warmuper.CloseAndWait() - // EnableWarmupCache toggle is a no-op now: HPH no longer holds a - // Go-side warmup cache. The warmer still pre-fetches via ctx.Branch / - // Account / Storage to populate sd.mem and OS page cache; the - // trie-side reads go through the same paths and hit the - // aggregator-scope BranchCache (for commitment) or page cache - // (accounts/storage). The flag plumbing is retained pending step 2c - // removal. - _ = warmup.EnableWarmupCache } err = updates.HashSort(ctx, warmuper, func(hashedKey, plainKey []byte, stateUpdate *Update) error { @@ -3029,8 +3009,6 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } -func (hph *HexPatriciaHashed) EnableWarmupCache(enable bool) { hph.enableWarmupCache = enable } - // SetBranchCache attaches a BranchCache for branch read-through and // write-through. Also propagates to the trie's BranchEncoder so encoder // writes update the cache via mark-dirty-then-Put (see CollectUpdate). @@ -3127,12 +3105,6 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// Cache always returns nil — HPH no longer holds a Go-side warmup -// cache. Retained as a stub so external callers (squeeze/fork-validation -// ClearWarmupCache plumbing) compile; they short-circuit on nil. -// Pending step 2c removal of the EnableWarmupCache plumbing chain. -func (hph *HexPatriciaHashed) Cache() *WarmupCache { return nil } - // verifyBranchCache, when true, makes branchFromCacheOrDB cross-check // every BranchCache hit against ctx.Branch and record a divergence on // the cache when the bytes disagree. Off by default — gate via env diff --git a/execution/commitment/warmup_cache.go b/execution/commitment/warmup_cache.go deleted file mode 100644 index d1e73d971a0..00000000000 --- a/execution/commitment/warmup_cache.go +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "fmt" - "sync" - "sync/atomic" - - "github.com/erigontech/erigon/common/maphash" -) - -type branchEntry struct { - // data is the canonical encoded form (with the leading 2-byte touch-map - // prefix). Always populated by PutBranch / PutBranchIfClean. - data []byte - - // Lazy-decoded form. Populated on the first GetBranchDecoded call for - // this entry; subsequent calls return the cached decode. decodeOnce - // ensures decode runs at most once per entry even under concurrent - // reads. - // - // Encoded form is the source of truth; decoded form is derived. When - // data is replaced (PutBranch overwrites), the new entry starts fresh - // — next decoded read re-derives from the new bytes. - decodeOnce sync.Once - cells [16]cell - cellsBitmap uint16 - decodedReady bool - decodeErr error - - isEvicted atomic.Bool - // dirty signals "the canonical store has been written to since this - // entry was populated; treat as stale until cleared." Read paths - // MAY return miss when dirty is set; write paths MUST set dirty. - // Used together with PutBranchIfClean to prevent late warmup writes - // from clobbering fresh fold writes (TOCTOU race documented in the - // reth-research §4 dirty-flag pattern). Ephemeral cache today does - // not have the race because warmup completes before fold begins; - // invariant is in place ahead of cross-block persistence work. - dirty atomic.Bool -} - -type accountEntry struct { - update *Update - isEvicted atomic.Bool -} - -type storageEntry struct { - update *Update - isEvicted atomic.Bool -} - -// WarmupCache stores pre-fetched data from the warmup phase to avoid -// repeated DB reads during trie processing. Uses maphash.Map for efficient -// byte slice key lookups without string allocations. -// -// Memory pooling strategy: -// -// 1. Don't add a sync.Pool of byte buffers between the read path and -// the cache. The read path already returns mmap-backed slices -// with zero allocation (seg.Getter.NextUncompressed for the -// CompressNone domains we cache); pooling Go-side buffers above -// us just adds a layer that gets copied into the cache and then -// GC'd. The cache copy IS the canonical allocation; there's -// nothing meaningful to pool on top of it. -// -// 2. The PutBranch / PutAccount / PutStorage copies exist for -// mmap-detachment, not for buffer reuse. Snapshot files can be -// unmapped under us during collation/squeeze; copying detaches -// the cache entry from that lifetime. If tempted to "skip the -// copy because the source is fresh anyway" — check whether the -// source is mmap-backed. For AccountsDomain / StorageDomain / -// CommitmentDomain values (CompressNone in state_schema.go) it -// is, and the copy is non-negotiable. For CodeDomain -// (CompressVals) the source is a freshly decompressed buffer and -// the copy is unnecessary; that's a separate narrow -// optimisation, don't generalise. -// -// 3. Don't shorten the cache lifetime to per-block "for safety." -// The structural win is the opposite: extend it across blocks so -// overlapping prefixes (root, contract trunks) get allocated -// once and reused. Today the cache is created per -// ComputeCommitment call (hex_patricia_hashed.go ~2780) and -// torn down at block end, so the same ~26 K branches on the -// SSTORE-bloat workload are re-fetched and re-allocated every -// block. Per-block lifetime is the dominant unmeasured cost; a -// single longer-lived cache gives you both lookup efficiency -// and memory efficiency by holding exactly one copy of each -// entry. Correctness for cross-block reuse needs an -// invalidation story for entries written this block — see the -// dirty-flag scaffolding (PutBranchIfClean / MarkBranchDirty) -// for the intended hook. -type WarmupCache struct { - branches *maphash.Map[*branchEntry] - accounts *maphash.Map[*accountEntry] - storage *maphash.Map[*storageEntry] - - // Observability counters. Updated by Get/Put paths; read by Stats(). - // Atomic so warmup-worker reads and main-trie reads don't race. - branchHits, branchMisses, branchEvicted atomic.Uint64 - branchBytesServed atomic.Uint64 - accountHits, accountMisses atomic.Uint64 - storageHits, storageMisses atomic.Uint64 -} - -// NewWarmupCache creates a new warmup cache instance. -func NewWarmupCache() *WarmupCache { - return &WarmupCache{ - branches: maphash.NewMap[*branchEntry](), - accounts: maphash.NewMap[*accountEntry](), - storage: maphash.NewMap[*storageEntry](), - } -} - -// PutBranch stores branch data in the cache. -func (c *WarmupCache) PutBranch(prefix []byte, data []byte) { - // Make a copy of the data to avoid issues with buffer reuse - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - - c.branches.Set(prefix, &branchEntry{data: dataCopy}) -} - -// PutBranchIfClean stores branch data in the cache only if no existing entry -// is marked dirty. Returns true on store, false if a dirty entry was present -// (indicating the canonical store has been updated since the caller last -// read, and the caller's data is potentially stale). -// -// Use from warmup-style writers that may race with fold writes — the fold -// path marks branches dirty before its own write completes, so a warmup -// worker that reads pre-fold then attempts to write post-fold will see -// dirty=true and skip the store. -// -// Today's call sites (warmup workers in HexPatriciaHashed.Process) do not -// race with fold because warmup completes before HashSort begins. Invariant -// is in place for future cross-block persistence work where warmup-style -// writes can outlive their parent Process. -func (c *WarmupCache) PutBranchIfClean(prefix []byte, data []byte) bool { - if existing, found := c.branches.Get(prefix); found && existing.dirty.Load() { - return false - } - dataCopy := make([]byte, len(data)) - copy(dataCopy, data) - c.branches.Set(prefix, &branchEntry{data: dataCopy}) - return true -} - -// MarkBranchDirty flags a branch entry as stale-until-cleared. The next -// PutBranchIfClean for this prefix will skip; reads (GetBranch / -// GetAndEvictBranch) currently return the entry regardless — the dirty -// signal is consumed only on the write path today. -// -// Use from fold/encoder paths that have decided to overwrite a branch but -// haven't yet captured the new bytes. The mark-dirty + later actual-write -// pattern is the deferred-encoding-friendly alternative to inline -// invalidate, motivated by the prototype investigation that found inline -// invalidate breaks update-in-place semantics under deferred encoding. -func (c *WarmupCache) MarkBranchDirty(prefix []byte) { - if entry, found := c.branches.Get(prefix); found { - entry.dirty.Store(true) - } -} - -// GetBranch retrieves branch data from the cache. -func (c *WarmupCache) GetBranch(prefix []byte) ([]byte, bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return nil, false - } - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.data, true -} - -// GetBranchDecoded retrieves the cached branch in decoded form. Lazy-decodes -// on first access for each entry; subsequent reads return the cached cells -// pointer without redoing the parse work. -// -// Returns the bitmap of present children plus a pointer to the populated -// cells array. The caller derives touchMap/afterMap from the bitmap based -// on its own context (deleted vs present-after) — the cache stores cells -// independent of that context so the same entry serves both readers. -// -// Returns ok=false on miss or eviction (same semantics as GetBranch). Also -// returns ok=false if the cached bytes fail to decode — in that case the -// caller should fall through to a re-read from the canonical store. -// -// The returned *[16]cell pointer aliases storage owned by the cache entry -// — the caller MUST NOT modify the cells in place. Read-only consumption -// is safe across concurrent GetBranchDecoded calls (decode runs at most -// once per entry; subsequent reads see the populated cells). -func (c *WarmupCache) GetBranchDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return 0, nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return 0, nil, false - } - entry.decodeOnce.Do(func() { - // PutBranch stores bytes WITH the leading 2-byte touch-map prefix; - // DecodeBranchInto consumes bytes WITHOUT it (matching the - // unfoldBranchNode call pattern that strips the prefix before - // decoding). - if len(entry.data) < 2 { - entry.decodeErr = fmt.Errorf("branch entry too short for touch-map prefix: %d bytes", len(entry.data)) - return - } - maps, err := DecodeBranchInto(entry.data[2:], false /* deleted derived per-caller */, &entry.cells) - if err != nil { - entry.decodeErr = err - return - } - entry.cellsBitmap = maps.Bitmap - entry.decodedReady = true - }) - if !entry.decodedReady { - // Decode failed — caller should re-read from canonical store. - // Don't count as hit (the entry is mechanically present but - // unusable). Don't increment misses either — the caller will - // observe ok=false and decide. - return 0, nil, false - } - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.cellsBitmap, &entry.cells, true -} - -// GetAndEvictBranch retrieves branch data and marks the entry as evicted in one operation. -func (c *WarmupCache) GetAndEvictBranch(prefix []byte) ([]byte, bool) { - entry, found := c.branches.Get(prefix) - if !found { - c.branchMisses.Add(1) - return nil, false - } - if entry.isEvicted.Load() { - c.branchEvicted.Add(1) - return nil, false - } - entry.isEvicted.Store(true) - c.branchHits.Add(1) - c.branchBytesServed.Add(uint64(len(entry.data))) - return entry.data, true -} - -// EvictBranch marks a branch entry as evicted without retrieving it. -func (c *WarmupCache) EvictBranch(prefix []byte) { - entry, found := c.branches.Get(prefix) - if found { - entry.isEvicted.Store(true) - } -} - -// PutAccount stores account data in the cache. -func (c *WarmupCache) PutAccount(plainKey []byte, update *Update) { - var updateCopy *Update - if update != nil { - updateCopy = update.Copy() - } - - c.accounts.Set(plainKey, &accountEntry{update: updateCopy}) -} - -// GetAccount retrieves account data from the cache. -func (c *WarmupCache) GetAccount(plainKey []byte) (*Update, bool) { - entry, found := c.accounts.Get(plainKey) - if !found || entry.isEvicted.Load() { - c.accountMisses.Add(1) - return nil, false - } - c.accountHits.Add(1) - return entry.update, true -} - -// GetAndEvictAccount retrieves account data and marks the entry as evicted in one operation. -// Returns the entry pointer allowing the caller to read the data before it's considered evicted. -func (c *WarmupCache) GetAndEvictAccount(plainKey []byte) *accountEntry { - entry, found := c.accounts.Get(plainKey) - if !found || entry.isEvicted.Load() { - return nil - } - entry.isEvicted.Store(true) - return entry -} - -// EvictAccount marks an account entry as evicted without retrieving it. -func (c *WarmupCache) EvictAccount(plainKey []byte) { - entry, found := c.accounts.Get(plainKey) - if found { - entry.isEvicted.Store(true) - } -} - -// PutStorage stores storage data in the cache. -func (c *WarmupCache) PutStorage(plainKey []byte, update *Update) { - var updateCopy *Update - if update != nil { - updateCopy = update.Copy() - } - - c.storage.Set(plainKey, &storageEntry{update: updateCopy}) -} - -// GetStorage retrieves storage data from the cache. -func (c *WarmupCache) GetStorage(plainKey []byte) (*Update, bool) { - entry, found := c.storage.Get(plainKey) - if !found || entry.isEvicted.Load() { - c.storageMisses.Add(1) - return nil, false - } - c.storageHits.Add(1) - return entry.update, true -} - -// GetAndEvictStorage retrieves storage data and marks the entry as evicted in one operation. -// Returns the entry pointer allowing the caller to read the data before it's considered evicted. -func (c *WarmupCache) GetAndEvictStorage(plainKey []byte) *storageEntry { - entry, found := c.storage.Get(plainKey) - if !found || entry.isEvicted.Load() { - return nil - } - entry.isEvicted.Store(true) - return entry -} - -// EvictStorage marks a storage entry as evicted without retrieving it. -func (c *WarmupCache) EvictStorage(plainKey []byte) { - entry, found := c.storage.Get(plainKey) - if found { - entry.isEvicted.Store(true) - } -} - -// EvictPlainKey evicts a key from both accounts and storage caches. -// Use this when you don't know if the key is an account or storage key. -func (c *WarmupCache) EvictPlainKey(plainKey []byte) { - if entry, found := c.accounts.Get(plainKey); found { - entry.isEvicted.Store(true) - } - if entry, found := c.storage.Get(plainKey); found { - entry.isEvicted.Store(true) - } -} - -// Clear clears all cached data and resets stats counters. -func (c *WarmupCache) Clear() { - c.branches = maphash.NewMap[*branchEntry]() - c.accounts = maphash.NewMap[*accountEntry]() - c.storage = maphash.NewMap[*storageEntry]() - c.ResetStats() -} - -// Stats returns a one-line summary of cache hit/miss counters. -// Format matches the per-block log line: branch, account, storage with -// hit-percentages and bytes served. Useful in commitment debug logs and -// for the per-Process LogCommitments line. -func (c *WarmupCache) Stats() string { - bh, bm, be := c.branchHits.Load(), c.branchMisses.Load(), c.branchEvicted.Load() - bb := c.branchBytesServed.Load() - ah, am := c.accountHits.Load(), c.accountMisses.Load() - sh, sm := c.storageHits.Load(), c.storageMisses.Load() - pct := func(hit, miss uint64) float64 { - total := hit + miss - if total == 0 { - return 0 - } - return 100.0 * float64(hit) / float64(total) - } - return fmt.Sprintf( - "branch hit=%d miss=%d evict=%d (%.1f%%, %.1f MiB) | acct hit=%d miss=%d (%.1f%%) | stor hit=%d miss=%d (%.1f%%)", - bh, bm, be, pct(bh, bm), float64(bb)/1024/1024, - ah, am, pct(ah, am), - sh, sm, pct(sh, sm), - ) -} - -// ResetStats zeros out all hit/miss/byte counters without touching cached data. -func (c *WarmupCache) ResetStats() { - c.branchHits.Store(0) - c.branchMisses.Store(0) - c.branchEvicted.Store(0) - c.branchBytesServed.Store(0) - c.accountHits.Store(0) - c.accountMisses.Store(0) - c.storageHits.Store(0) - c.storageMisses.Store(0) -} diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index cb6997ebd58..807f6264eed 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -55,12 +55,11 @@ type TrieContextFactory func() (PatriciaContext, func()) // WarmupConfig contains configuration for pre-warming MDBX page cache // during commitment processing. type WarmupConfig struct { - Enabled bool - EnableWarmupCache bool // If true, cache warmed data for use during trie processing - CtxFactory TrieContextFactory - NumWorkers int - MaxDepth int - LogPrefix string + Enabled bool + CtxFactory TrieContextFactory + NumWorkers int + MaxDepth int + LogPrefix string } const WarmupMaxDepth = 128 // covers full key paths for both account keys (64 nibbles) and storage keys (128 nibbles) @@ -85,9 +84,6 @@ type Warmuper struct { // Worker group g *errgroup.Group - // Cache for storing warmed data to be used during trie processing - cache *WarmupCache - // Stats keysProcessed atomic.Uint64 startTime time.Time @@ -105,7 +101,7 @@ type warmupWorkItem struct { // NewWarmuper creates a new Warmuper instance. func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { ctx, cancel := context.WithCancel(ctx) - w := &Warmuper{ + return &Warmuper{ ctx: ctx, cancel: cancel, ctxFactory: cfg.CtxFactory, @@ -113,65 +109,35 @@ func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { numWorkers: cfg.NumWorkers, logPrefix: cfg.LogPrefix, } - if cfg.EnableWarmupCache { - w.cache = NewWarmupCache() - } - return w -} - -// Cache returns the warmup cache, or nil if caching is disabled. -func (w *Warmuper) Cache() *WarmupCache { - return w.cache } -// branchFromCacheOrDB reads branch data from cache if available, otherwise from DB and caches it. +// branchFromCacheOrDB reads branch data via ctx.Branch. The Go-side +// warmup cache that previously sat above this has been removed; the +// aggregator-scope BranchCache (reached through SD.GetLatest) is the +// only branch cache. The function is kept as a thin wrapper to +// preserve the warmer-side error handling shape. func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ([]byte, error) { - if w.cache != nil { - if data, found := w.cache.GetBranch(prefix); found { - return data, nil - } - } branchData, _, err := trieCtx.Branch(prefix) - if err != nil { - return nil, err - } - if w.cache != nil && len(branchData) > 0 { - w.cache.PutBranch(prefix, branchData) - } - return branchData, nil + return branchData, err } -// accountFromCacheOrDB reads account data from cache if available, otherwise from DB and caches it. +// accountFromCacheOrDB reads account data via ctx.Account. No Go-side +// caching; the read populates the OS page cache as a side effect. func (w *Warmuper) accountFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - if w.cache != nil { - if update, found := w.cache.GetAccount(plainKey); found { - return update, nil - } - } update, err := trieCtx.Account(plainKey) if err != nil { return nil, err } - if w.cache != nil { - w.cache.PutAccount(plainKey, update) - } return update, nil } -// storageFromCacheOrDB reads storage data from cache if available, otherwise from DB and caches it. +// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side +// caching; the read populates the OS page cache as a side effect. func (w *Warmuper) storageFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - if w.cache != nil { - if update, found := w.cache.GetStorage(plainKey); found { - return update, nil - } - } update, err := trieCtx.Storage(plainKey) if err != nil { return nil, err } - if w.cache != nil { - w.cache.PutStorage(plainKey, update) - } return update, nil } diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 6dc31c94eca..af559768993 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -205,8 +205,6 @@ func ExecV3(ctx context.Context, doms.EnableParaTrieDB(cfg.db) doms.EnableTrieWarmup(true) - // Do it only for chain-tip blocks! - doms.EnableWarmupCache(!isApplyingBlocks) doms.SetDeferCommitmentUpdates(false) // Enable deferred commitment updates for fork validation and parallel initial sync. // Deferred updates batch commitment calculations to block boundaries rather than From b99c1926440046f9ae21d5f94e0205e32086c745 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:16:07 +0000 Subject: [PATCH 040/120] commitment: mark BranchCache dirty in CollectDeferredUpdate too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CollectUpdate (immediate path) calls be.branchCache.MarkDirty(prefix) before encoding so concurrent warmer-style writers see the dirty flag and skip via PutIfClean. CollectDeferredUpdate (deferred path, used by parallel commitment) was missing the same call — meaning a prefix queued for deferred apply could be overwritten by a stale PutIfClean from the warmup path between enqueue and eventual ApplyDeferredBranchUpdates write. Add the matching MarkDirty call so both encoder entry points consistently invalidate the cache. SD.Flush stays the canonical update site (Invalidate + Put on each flushed prefix); the encoder hooks are the only other places branch writes originate. Step 3 of the WarmupCache deletion / BranchCache consolidation. --- execution/commitment/commitment.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index ecaf7b4b107..612b06080f9 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -657,6 +657,17 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } + // Mark BranchCache entry dirty BEFORE the deferred enqueue. Mirrors + // the CollectUpdate path so that both immediate and deferred encoder + // entries consistently invalidate the cache. Concurrent warmer-style + // writers calling PutIfClean for this prefix between now and the + // eventual ApplyDeferredBranchUpdates write see the dirty flag and + // skip — preventing a stale read from racing the deferred write. + // See branch_cache.go's Concurrency Contract. + if be.branchCache != nil { + be.branchCache.MarkDirty(prefix) + } + prev, _, err := ctx.Branch(prefix) if err != nil { return err From 80ea439f12df79a9ab38b8247946e37c40ad3687 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 13:41:49 +0000 Subject: [PATCH 041/120] =?UTF-8?q?commitment/warmuper:=20scope=20warmer?= =?UTF-8?q?=20to=20branches=20only=20=E2=80=94=20drop=20account/storage=20?= =?UTF-8?q?prefetch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch warm-up's scope is branches. Account/storage data is downstream of the executor; if a leaf hash needs it during fold, the data is either (a) memoized as the cell's stateHash, (b) carried in the Updates buffer from the executor, or (c) faulted on the trie's slow path — which signals a memoization gap to fix at the trie layer, not to paper over with a separate prefetcher. The warmer was reading sibling cell addresses out of branch bytes (via extractBranchCellAddresses) and pre-fetching the underlying accounts/storage from BTree to warm the OS page cache. Evidence from the SSTORE-bloat bench shows this work is unused on the workloads we measure: disk_sto=0 / disk_acc=0 in the cache-fp log means the trie's fold-time fall-through never fires. The path cells that *are* extracted by the helper are already warm from the executor's reads (EIP-2200 SSTORE gas calc on the same slots); sibling cells with memoized stateHashes are correctly filtered out. Removes: - Warmuper.warmupKey: cellAccounts/cellStorages prefetch loops - Warmuper.accountFromCacheOrDB / storageFromCacheOrDB - extractBranchCellAddresses (only caller goes away; skipCellFields stays — warmuper bitmap walk + trie_reader use it) The disk_sto / disk_acc counters stay in place as the canary: any future workload that drives them non-zero is a signal that something needs the underlying data, which is itself a question for the trie / IBS integration layer, not the warmer. Step 4 of the WarmupCache deletion / BranchCache consolidation. Verified: build green; verify-bench.sh expected to PASS (the path that was being removed is unused on the bench). --- execution/commitment/hex_patricia_hashed.go | 111 -------------------- execution/commitment/warmuper.go | 37 ++----- 2 files changed, 9 insertions(+), 139 deletions(-) diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index da5869765be..ce4bb76cd84 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -588,117 +588,6 @@ func skipCellFields(data []byte, pos int, fieldBits byte) int { return pos } -// extractBranchCellAddresses extracts account/storage addresses from branch data. -// pathNibble is the nibble on the update path (-1 if none) - its addresses are always -// extracted since that cell will have stateHash cleared during update. -// Sibling cells with stateHash (memoized) are skipped. -func extractBranchCellAddresses(branchData []byte, pathNibble int) (accountAddrs [][]byte, storageAddrs [][]byte) { - if len(branchData) < 4 { - return nil, nil - } - // Skip touch map (first 2 bytes) - branchData = branchData[2:] - bitmap := binary.BigEndian.Uint16(branchData[0:2]) - pos := 2 - - // Iterate over all cells in the branch - for bitset := bitmap; bitset != 0; { - bit := bitset & -bitset - nibble := bits.TrailingZeros16(bit) - bitset ^= bit - - if pos >= len(branchData) { - break - } - fieldBits := branchData[pos] - pos++ - - // Cell on update path will have stateHash cleared - always extract its addresses. - // Sibling cells with stateHash (bit 4) are memoized - skip them. - isOnPath := nibble == pathNibble - hasMemoizedHash := fieldBits&16 != 0 - shouldExtract := isOnPath || !hasMemoizedHash - - // Parse each field - // extension (bit 0) - if fieldBits&1 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - - // accountAddr (bit 1) - if fieldBits&2 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n - if l > 0 && pos+int(l) <= len(branchData) { - if shouldExtract { - addr := make([]byte, l) - copy(addr, branchData[pos:pos+int(l)]) - accountAddrs = append(accountAddrs, addr) - } - pos += int(l) - } - } - - // storageAddr (bit 2) - if fieldBits&4 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n - if l > 0 && pos+int(l) <= len(branchData) { - if shouldExtract { - addr := make([]byte, l) - copy(addr, branchData[pos:pos+int(l)]) - storageAddrs = append(storageAddrs, addr) - } - pos += int(l) - } - } - - // hash (bit 3) - if fieldBits&8 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - - // stateHash (bit 4) - if fieldBits&16 != 0 { - if pos >= len(branchData) { - break - } - l, n := binary.Uvarint(branchData[pos:]) - if n <= 0 { - break - } - pos += n + int(l) - } - } - return accountAddrs, storageAddrs -} - func (cell *cell) accountForHashing(buffer []byte, storageRootHash common.Hash) int { balanceBytes := 0 if !cell.Balance.LtUint64(128) { diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index 807f6264eed..f71a73fa312 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -121,26 +121,6 @@ func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ( return branchData, err } -// accountFromCacheOrDB reads account data via ctx.Account. No Go-side -// caching; the read populates the OS page cache as a side effect. -func (w *Warmuper) accountFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - update, err := trieCtx.Account(plainKey) - if err != nil { - return nil, err - } - return update, nil -} - -// storageFromCacheOrDB reads storage data via ctx.Storage. No Go-side -// caching; the read populates the OS page cache as a side effect. -func (w *Warmuper) storageFromCacheOrDB(trieCtx PatriciaContext, plainKey []byte) (*Update, error) { - update, err := trieCtx.Storage(plainKey) - if err != nil { - return nil, err - } - return update, nil -} - // Start initializes and starts the warmup workers. func (w *Warmuper) Start() { if w.started.Swap(true) { @@ -201,14 +181,15 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep } nextNibble := int(hashedKey[depth]) - // Extract and prefetch account/storage addresses to warm page cache - cellAccounts, cellStorages := extractBranchCellAddresses(branchData, nextNibble) - for _, addr := range cellAccounts { - _, _ = w.accountFromCacheOrDB(trieCtx, addr) - } - for _, addr := range cellStorages { - _, _ = w.storageFromCacheOrDB(trieCtx, addr) - } + // Account/storage prefetch removed: the warmer's scope is branch + // warm-up only. If a leaf hash needs underlying account/storage + // data during fold, that data is either (a) memoized as the + // cell's stateHash, (b) carried in the Updates buffer from the + // executor, or (c) faulted on the trie's own slow path — + // signalling a memoization gap that wants fixing at the trie + // layer, not papered over by a separate prefetcher. Counters + // disk_sto / disk_acc on the [commitment][cache-fp] log line + // expose any such fall-through. branchData = branchData[2:] // skip touch map From 614fb7bcc188884c46c206642294dd8b657d1f0e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 14:18:22 +0000 Subject: [PATCH 042/120] state, commitment: add unique-prefix file-read counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface read amplification on the cache-fp log: alongside the existing per-domain FileReadCount (total file reads), track UniqueFileReadCount (distinct prefixes). The ratio FileReadCount / UniqueFileReadCount tells us how often the same prefix was re-read from the file layer. On the SSTORE-bloat workload, files_comm=26K commitment file reads for ~5,910 slots in a single contract is suspicious: the account-prefix path is constant for all 5,910 slots and the storage subtree's structural branch count is ~3,000-3,500. ~8x amplification; uniq_comm tells us how much of that is distinct prefix coverage vs repeat reads on the same prefix. DomainMetrics gets a sync.Map dedup keyed by ":" — process-cumulative, gated by the existing dbg.KVReadLevelledMetrics flag so production isn't charged when metrics are off. UpdateFileReadsUnique replaces UpdateFileReads at the two MeteredGetLatest call sites (aggregator.go + domain.go). --- db/state/aggregator.go | 7 +- db/state/changeset/state_changeset.go | 84 +++++++++++++++++++ db/state/domain.go | 2 +- .../commitmentdb/commitment_context.go | 11 ++- 4 files changed, 101 insertions(+), 3 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 93ff05ed6d0..387e4f4a906 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2503,7 +2503,12 @@ func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxSte return nil, kv.Step(0), false, err } if metrics != nil && dbg.KVReadLevelledMetrics { - metrics.UpdateFileReads(domain, start) + // UpdateFileReadsUnique tracks both total reads and distinct + // prefixes. The ratio FileReadCount / UniqueFileReadCount is + // the read amplification factor — useful when investigating + // whether the same prefixes are being re-read from file layer + // (cache misses on hot prefixes). + metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) return v, kv.Step(fileEndTxNum / at.StepSize()), found, err diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 057eba818db..ff5d2fab30b 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -464,12 +464,35 @@ type DomainIOMetrics struct { DbReadDuration time.Duration FileReadCount int64 FileReadDuration time.Duration + // UniqueFileReadCount tracks distinct prefixes that ever fell through + // to a file read (process-cumulative). The ratio FileReadCount / + // UniqueFileReadCount is the read amplification factor: how many + // times each unique prefix was re-read from the file layer (cache + // misses on the same prefix). Updated by UpdateFileReadsUnique; + // gated by dbg.KVReadLevelledMetrics same as FileReadCount. + UniqueFileReadCount int64 + + // StateCache hit/miss tracks the SharedDomains.stateCache layer + // specifically (the per-execution Account/Storage/Code cache), distinct + // from CacheReadCount which counts sd.mem and sd.parent.mem hits. + // Hit means stateCache.Get returned ok (we skipped MDBX+files). + // Miss means stateCache.Get returned !ok and we fell through to aggTx. + StateCacheHitCount int64 + StateCacheMissCount int64 } type DomainMetrics struct { sync.RWMutex DomainIOMetrics Domains map[kv.Domain]*DomainIOMetrics + + // seenFileReads is a process-cumulative dedup set for + // UpdateFileReadsUnique. Keys are (domain.String() + prefixBytes); + // LoadOrStore on each read decides whether the prefix is new and + // therefore contributes to UniqueFileReadCount. Held outside the + // mutex (sync.Map handles its own concurrency) so the unique check + // doesn't serialise with other metric updates. + seenFileReads sync.Map } func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { @@ -506,6 +529,28 @@ func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { } } +func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { + dm.Lock() + defer dm.Unlock() + dm.StateCacheHitCount++ + if d, ok := dm.Domains[domain]; ok { + d.StateCacheHitCount++ + } else { + dm.Domains[domain] = &DomainIOMetrics{StateCacheHitCount: 1} + } +} + +func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { + dm.Lock() + defer dm.Unlock() + dm.StateCacheMissCount++ + if d, ok := dm.Domains[domain]; ok { + d.StateCacheMissCount++ + } else { + dm.Domains[domain] = &DomainIOMetrics{StateCacheMissCount: 1} + } +} + func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { dm.Lock() defer dm.Unlock() @@ -522,3 +567,42 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } } } + +// UpdateFileReadsUnique records a file read while also tracking whether +// the prefix has been seen before for this domain. Same gate as +// UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when +// the read-amplification ratio (FileReadCount / UniqueFileReadCount) +// is wanted on the metric output. The key bytes are copied into the +// internal dedup map; do not mutate after the call. +func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + // Composite ":" so two domains can hold the same prefix + // shape (commitment compact-encoded paths vs accounts plain addresses) + // without colliding. + domainKey := domain.String() + ":" + string(key) + _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) + + dm.Lock() + defer dm.Unlock() + dm.FileReadCount++ + readDuration := time.Since(start) + dm.FileReadDuration += readDuration + if !alreadySeen { + dm.UniqueFileReadCount++ + } + if d, ok := dm.Domains[domain]; ok { + d.FileReadCount++ + d.FileReadDuration += readDuration + if !alreadySeen { + d.UniqueFileReadCount++ + } + } else { + newD := &DomainIOMetrics{ + FileReadCount: 1, + FileReadDuration: readDuration, + } + if !alreadySeen { + newD.UniqueFileReadCount = 1 + } + dm.Domains[domain] = newD + } +} diff --git a/db/state/domain.go b/db/state/domain.go index d7b52ff7fbe..d9158f5ce42 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1641,7 +1641,7 @@ func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics v, foundInFile, _, endTxNum, err := dt.getLatestFromFiles(key, 0) if metrics != nil && dbg.KVReadLevelledMetrics { - metrics.UpdateFileReads(dt.name, start) + metrics.UpdateFileReadsUnique(dt.name, key, start) } if err != nil { diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index f1b488ef452..410557706bd 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -362,19 +362,24 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // (branch reads), Storage (value loads), Account, Code. // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 + var aUniq, sUniq, cUniq, mUniq int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { aFiles = d.FileReadCount + aUniq = d.UniqueFileReadCount } if d := m.Domains[kv.StorageDomain]; d != nil { sFiles = d.FileReadCount + sUniq = d.UniqueFileReadCount } if d := m.Domains[kv.CodeDomain]; d != nil { cFiles = d.FileReadCount + cUniq = d.UniqueFileReadCount } if d := m.Domains[kv.CommitmentDomain]; d != nil { mFiles = d.FileReadCount + mUniq = d.UniqueFileReadCount } m.RUnlock() } @@ -401,7 +406,11 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "files_acc", aFiles, "files_sto", sFiles, "files_code", cFiles, - "files_comm", mFiles) + "files_comm", mFiles, + "uniq_acc", aUniq, + "uniq_sto", sUniq, + "uniq_code", cUniq, + "uniq_comm", mUniq) } } } From 4b95d45e6b510125f81764a319de9d17f80931c2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 14:44:58 +0000 Subject: [PATCH 043/120] state, commitment: add unique-prefix length histogram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface byte-length distribution of distinct prefixes that fall through to a file read. Buckets: 1B (depth 0-1, root) 2-4B (depth 2-7) 5-8B (depth 8-15) 9-16B (depth 16-31) 17-32B (depth 32-63) 33-64B (depth 64-127) >64B (out of range) Tells us where the 25K commitment reads concentrate. If they cluster in the 33-64B / >64B range, they're storage-subtree leaf-parents (per-slot, structurally irreducible). If they cluster shallower (account-trunk / contract-trunk), there's a caching lever to pin those prefixes — e.g. an account-root cache over the first X nibbles per contract. Compact-encoded prefix length is depth/2 + 1, with the parity flag in the first nibble (HP encoding from Yellow Paper). See nibbles.HexToCompact. --- db/state/changeset/state_changeset.go | 39 +++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index ff5d2fab30b..8c03b394491 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -471,6 +471,20 @@ type DomainIOMetrics struct { // misses on the same prefix). Updated by UpdateFileReadsUnique; // gated by dbg.KVReadLevelledMetrics same as FileReadCount. UniqueFileReadCount int64 + // UniqueLenBuckets is the byte-length distribution of distinct + // prefixes seen by UpdateFileReadsUnique. The compact-encoded + // commitment prefix carries depth-dependent length: + // 0 : 1 byte (root / depth 0) + // 1 : 2-4 bytes + // 2 : 5-8 bytes + // 3 : 9-16 bytes + // 4 : 17-32 bytes + // 5 : 33-64 bytes + // 6 : 65+ bytes (out of range for storage paths) + // Used to investigate depth distribution of touched prefixes — + // e.g., are the 25K commitment reads mostly leaf-parents (deep) + // or trunk (shallow)? + UniqueLenBuckets [7]int64 // StateCache hit/miss tracks the SharedDomains.stateCache layer // specifically (the per-execution Account/Storage/Code cache), distinct @@ -568,6 +582,27 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } } +// lenBucket maps a prefix byte-length to its UniqueLenBuckets index. +// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. +func lenBucket(n int) int { + switch { + case n <= 1: + return 0 + case n <= 4: + return 1 + case n <= 8: + return 2 + case n <= 16: + return 3 + case n <= 32: + return 4 + case n <= 64: + return 5 + default: + return 6 + } +} + // UpdateFileReadsUnique records a file read while also tracking whether // the prefix has been seen before for this domain. Same gate as // UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when @@ -580,6 +615,7 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta // without colliding. domainKey := domain.String() + ":" + string(key) _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) + bucket := lenBucket(len(key)) dm.Lock() defer dm.Unlock() @@ -588,12 +624,14 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta dm.FileReadDuration += readDuration if !alreadySeen { dm.UniqueFileReadCount++ + dm.UniqueLenBuckets[bucket]++ } if d, ok := dm.Domains[domain]; ok { d.FileReadCount++ d.FileReadDuration += readDuration if !alreadySeen { d.UniqueFileReadCount++ + d.UniqueLenBuckets[bucket]++ } } else { newD := &DomainIOMetrics{ @@ -602,6 +640,7 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta } if !alreadySeen { newD.UniqueFileReadCount = 1 + newD.UniqueLenBuckets[bucket] = 1 } dm.Domains[domain] = newD } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 410557706bd..57a835a76c3 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -363,6 +363,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 var aUniq, sUniq, cUniq, mUniq int64 + var mLens [7]int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { @@ -380,9 +381,13 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context if d := m.Domains[kv.CommitmentDomain]; d != nil { mFiles = d.FileReadCount mUniq = d.UniqueFileReadCount + mLens = d.UniqueLenBuckets } m.RUnlock() } + // commLens compact form: "1B:N0|2-4B:N1|5-8B:N2|9-16B:N3|17-32B:N4|33-64B:N5|>64B:N6" + commLens := fmt.Sprintf("1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33-64B:%d|>64B:%d", + mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6]) log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), @@ -410,7 +415,8 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "uniq_acc", aUniq, "uniq_sto", sUniq, "uniq_code", cUniq, - "uniq_comm", mUniq) + "uniq_comm", mUniq, + "comm_lens", commLens) } } } From d566e9b163c21d6327926cb3713c3fa0af679e3e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:22:26 +0000 Subject: [PATCH 044/120] state, commitment: refine length histogram inside storage subtree The prior 33-64B bucket spanned all storage-subtree depths (64-127), too coarse to distinguish per-contract storage-trunk reads (where pinning would help) from leaf-parent reads (where it doesn't). Sub-divide into four buckets: 33B : storage subtree root for a contract (depth 64) 34-36B : storage subtree top, depth 65-70 (per-contract trunk) 37-44B : mid storage, depth 71-86 45-64B : leaf-parents and deep paths, depth 87-127 If most of the 25K commitment reads concentrate at 33B / 34-36B, the per-contract storage-trunk reads multiply across all touched slots within a contract; pinning the contract's first few storage- subtree branches (the user's 'storage root trunk cache for big accounts' idea) would absorb them. If reads concentrate at 45-64B, paths are inherently per-slot and pinning the trunk wouldn't help. --- db/state/changeset/state_changeset.go | 43 ++++++++++++------- .../commitmentdb/commitment_context.go | 13 ++++-- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 8c03b394491..0e30f687897 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -473,18 +473,23 @@ type DomainIOMetrics struct { UniqueFileReadCount int64 // UniqueLenBuckets is the byte-length distribution of distinct // prefixes seen by UpdateFileReadsUnique. The compact-encoded - // commitment prefix carries depth-dependent length: - // 0 : 1 byte (root / depth 0) - // 1 : 2-4 bytes - // 2 : 5-8 bytes - // 3 : 9-16 bytes - // 4 : 17-32 bytes - // 5 : 33-64 bytes - // 6 : 65+ bytes (out of range for storage paths) - // Used to investigate depth distribution of touched prefixes — - // e.g., are the 25K commitment reads mostly leaf-parents (deep) - // or trunk (shallow)? - UniqueLenBuckets [7]int64 + // prefix length is 1 + ⌈depth/2⌉ bytes (HP encoding), so byte + // length maps to trie depth as follows: + // 0 : 1 byte (depth 0-1, root) + // 1 : 2-4 bytes (depth 2-7, top trunk) + // 2 : 5-8 bytes (depth 8-15) + // 3 : 9-16 bytes (depth 16-31) + // 4 : 17-32 bytes (depth 32-63, near account leaf) + // 5 : 33 bytes (depth 64, storage subtree root) + // 6 : 34-36 bytes (depth 65-70, storage subtree top) + // 7 : 37-44 bytes (depth 71-86, mid storage) + // 8 : 45-64 bytes (depth 87-127, leaf-parents and deep) + // 9 : 65+ bytes (out of range) + // Used to localise where the per-block file reads concentrate — + // e.g., are the 25K commitment reads on bloat workload + // storage-subtree-trunk (where per-contract pinning helps) or + // leaf-parents (where it doesn't)? + UniqueLenBuckets [10]int64 // StateCache hit/miss tracks the SharedDomains.stateCache layer // specifically (the per-execution Account/Storage/Code cache), distinct @@ -583,7 +588,9 @@ func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { } // lenBucket maps a prefix byte-length to its UniqueLenBuckets index. -// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. +// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets +// 5-8 sub-divide the storage subtree (depths 64-127) so per-contract +// trunk vs deep leaf-parent reads are distinguishable. func lenBucket(n int) int { switch { case n <= 1: @@ -596,10 +603,16 @@ func lenBucket(n int) int { return 3 case n <= 32: return 4 - case n <= 64: + case n == 33: return 5 - default: + case n <= 36: return 6 + case n <= 44: + return 7 + case n <= 64: + return 8 + default: + return 9 } } diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 57a835a76c3..a8bcf782dff 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -363,7 +363,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // All cumulative; deltas via successive lines. var aFiles, sFiles, cFiles, mFiles int64 var aUniq, sUniq, cUniq, mUniq int64 - var mLens [7]int64 + var mLens [10]int64 if m := sdc.sharedDomains.Metrics(); m != nil { m.RLock() if d := m.Domains[kv.AccountsDomain]; d != nil { @@ -385,9 +385,14 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } m.RUnlock() } - // commLens compact form: "1B:N0|2-4B:N1|5-8B:N2|9-16B:N3|17-32B:N4|33-64B:N5|>64B:N6" - commLens := fmt.Sprintf("1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33-64B:%d|>64B:%d", - mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6]) + // commLens shows depth distribution. Buckets within the storage + // subtree (33B+) are sub-divided to distinguish per-contract + // storage trunk (33B / 34-36B) from leaf-parent depths + // (45-64B). See state_changeset.go UniqueLenBuckets. + commLens := fmt.Sprintf( + "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", + mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], + mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), From 65fbd1422a4f487b7e031704e3ab56c59a939102 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 15:46:54 +0000 Subject: [PATCH 045/120] state/changeset: trunk-probe log to identify hot contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gated by TRUNK_PROBE=true. On first sight of a depth-64 commitment prefix (33-byte: 0x00 + 32 bytes of contract hash), log the contract hash. The trailing 32 bytes ARE keccak256(addr) and identify which contracts have storage activity in the workload. Used to pick the bloat contract for the per-contract storage-trunk-pin prototype. PIN_CONTRACT_TRUNKS env var is also threaded through the launcher (no consumer yet — placeholder for the next commit that adds the PinEntry mechanism). --- db/state/changeset/state_changeset.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 0e30f687897..04b0667a95d 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -18,6 +18,7 @@ package changeset import ( "encoding/binary" + "encoding/hex" "fmt" "math" "strings" @@ -27,11 +28,19 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/length" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/execution/types/accounts" ) +// logTrunkProbe is set by TRUNK_PROBE=true to log the keccak256 hash of +// each contract whose storage subtree root (depth-64 commitment prefix, +// 33 bytes) is read from the file layer for the first time. The bytes +// after the 0x00 flag byte ARE the contract's keccak256(addr), which +// is what PIN_CONTRACT_TRUNKS expects in its hex form. +var logTrunkProbe = dbg.EnvBool("TRUNK_PROBE", false) + type StateChangeSet struct { Diffs [kv.DomainLen]kv.DomainDiff // there are 4 domains of state changes } @@ -630,6 +639,13 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) bucket := lenBucket(len(key)) + // Trunk-probe log: depth-64 commitment prefix on first sight. + // Format key as 0x00 || keccak256(addr); the trailing 32 bytes are + // the value to feed to PIN_CONTRACT_TRUNKS for that contract. + if logTrunkProbe && !alreadySeen && bucket == 5 && domain == kv.CommitmentDomain && len(key) == 33 && key[0] == 0x00 { + log.Info("[trunk-probe] new depth-64 commitment prefix", "contract_hash", hex.EncodeToString(key[1:33])) + } + dm.Lock() defer dm.Unlock() dm.FileReadCount++ From c07a6881e647d75d9c211196994dd10b96ed994e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 8 May 2026 19:13:38 +0000 Subject: [PATCH 046/120] commitment: surface pinned-tier counters on cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pin_hit / pin_miss / pin_count fields to the [commitment][cache-fp] log line. The pinnedHits / pinnedMisses atomic counters on BranchCache existed but were not surfaced anywhere — without them the trunk-pin prototype is invisible (no way to tell whether the pinned tier is being used during the bench). Also add a PinnedStats() accessor and extend Stats() to include the pinned tier so the one-line summary reports all three tiers. This is the prerequisite instrumentation step before re-running the trunk-pin bench. With it, an outcome of "files_comm unchanged but pin_hit > 20K" tells us the pin is hit and trie compute uses the cached bytes; "files_comm unchanged AND pin_hit ~ 0" tells us the pin tier is bypassed (likely prefix encoding mismatch). No behaviour change. --- execution/commitment/branch_cache.go | 14 +++++++++++--- .../commitment/commitmentdb/commitment_context.go | 6 +++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index e6701db4514..1f138504a11 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -553,6 +553,7 @@ func (c *BranchCache) Fingerprint() uint64 { // per-Process log lines can compose them. func (c *BranchCache) Stats() string { rh, rm := c.rootHits.Load(), c.rootMisses.Load() + ph, pm := c.pinnedHits.Load(), c.pinnedMisses.Load() th, tm := c.tailHits.Load(), c.tailMisses.Load() bb := c.bytesServed.Load() pct := func(hit, miss uint64) float64 { @@ -563,15 +564,22 @@ func (c *BranchCache) Stats() string { return 100.0 * float64(hit) / float64(total) } return fmt.Sprintf( - "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) | served %.1f MiB | tail entries=%d | divergences=%d", + "branch-cache root hit=%d miss=%d (%.1f%%) | pin hit=%d miss=%d (%.1f%%) entries=%d | tail hit=%d miss=%d (%.1f%%) entries=%d | served %.1f MiB | divergences=%d", rh, rm, pct(rh, rm), - th, tm, pct(th, tm), + ph, pm, pct(ph, pm), c.pinned.Len(), + th, tm, pct(th, tm), c.tail.Len(), float64(bb)/1024/1024, - c.tail.Len(), c.verifyDivergences.Load(), ) } +// PinnedStats returns the pinned-tier hit/miss/entries counters. Used +// by the cache-fp log to expose pin effectiveness for the trunk-pin +// prototype debug. +func (c *BranchCache) PinnedStats() (hits, misses uint64, entries int) { + return c.pinnedHits.Load(), c.pinnedMisses.Load(), c.pinned.Len() +} + // VerifyDivergences returns the number of cache-vs-canonical divergences // recorded since the last Clear. Non-zero indicates a cache lifecycle // invariant has been violated (a cached entry no longer matches the diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index a8bcf782dff..5a97bceed4b 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -393,6 +393,7 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) + pinHits, pinMisses, pinEntries := bc.PinnedStats() log.Info("[commitment][cache-fp]", "block", blockNum, "root", hex.EncodeToString(rootHash), @@ -421,7 +422,10 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context "uniq_sto", sUniq, "uniq_code", cUniq, "uniq_comm", mUniq, - "comm_lens", commLens) + "comm_lens", commLens, + "pin_hit", pinHits, + "pin_miss", pinMisses, + "pin_count", pinEntries) } } } From a62cbaf98383f3aa5d8a3251f6ed784369ceb81e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 11:56:48 +0000 Subject: [PATCH 047/120] db/state, db/kv: add TemporalMemBatch.FlushWithCallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds FlushWithCallback (interface + concrete) for atomic flush + cache update. The cache callback fires for each latest (key, value, step) in the requested domain BEFORE the underlying Flush, all under latestStateLock so no observer sees a window where a key is missing from both the sd write buffer and the downstream cache. Used by SharedDomains.Flush to keep an external BranchCache in sync with commitment-domain writes. The MDBX commit provides db-side atomicity independently; the cache-first ordering keeps readers safe during the window between callback completion and MDBX write durably landing. Required by the WarmupCache-deletion sequence (commit ded378f789) which introduced the FlushWithCallback call site without defining the method itself — perf-stack at f0ba88aae2 didn't compile from source; the bench binary was built from a state with this method inlined and never committed. This commit reconstructs the method from the call-site contract. Verified: cold-cgroup d=68 SSTORE-bloat bench reproduces perf-stack baseline byte-for-byte (took=1.30 s, pin_count=69905, pin_hit=42216, pin_miss=104974, files_comm=16350, divergences=0). --- db/kv/kv_interface.go | 1 + db/state/temporal_mem_batch.go | 58 ++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e2836f786f3..16a76e0ff4e 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,6 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error + FlushWithCallback(ctx context.Context, tx RwTx, domain Domain, cb func(k []byte, v []byte, step Step)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 15e4c482d73..5ce7e1c8158 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -762,12 +762,53 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { return nil } -func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { +// FlushWithCallback updates a downstream cache via cb for each +// (key, value, step) tuple in domain, then flushes the mem-batch to tx. +// Cache-update is ordered BEFORE the MDBX flush so that during the +// MDBX write window the cache still holds the entry — a reader that +// goes (sd write buffer → cache → MDBX) never observes a key missing +// from all three sources. The MDBX commit provides db-side atomicity +// independently. +// +// Used by SharedDomains.Flush to keep an external BranchCache in sync +// with commitment-domain writes. +// +// cb is called under latestStateLock so the snapshot it sees matches +// the in-memory state at flush time. Flush itself runs under the same +// lock to prevent concurrent DomainPut from interleaving with the +// snapshot/cache-update/flush sequence. +// +// If Flush fails, the cache has already been updated for this batch. +// That's the same invariant the cache maintains for any other write +// path — values may exist in the cache before they reach disk; a +// retry of Flush re-applies them. +func (sd *TemporalMemBatch) FlushWithCallback( + ctx context.Context, tx kv.RwTx, domain kv.Domain, + cb func(k []byte, v []byte, step kv.Step), +) error { + sd.latestStateLock.Lock() + defer sd.latestStateLock.Unlock() + + // dir is unset on entries written via DomainPut/DomainDel (always + // default zero); the latest history entry per key is authoritative. + // data may be nil/empty for deletes — caller's cb is expected to + // distinguish (see SharedDomains.Flush, which Invalidate's on + // len(v)==0 and Put's otherwise). + for keyStr, history := range sd.domains[domain] { + if len(history) == 0 { + continue + } + latest := history[len(history)-1] + cb([]byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + } + + return sd.flushLocked(ctx, tx) +} + +// flushLocked is the body of Flush, factored so FlushWithCallback can +// run it inside latestStateLock without re-acquiring. +func (sd *TemporalMemBatch) flushLocked(ctx context.Context, tx kv.RwTx) error { if sd.unwindChangesetRaw != nil { - // Replay against the RAW (step-preserving) changeset — the collapsed - // unwindChangeset would lose every (key, step) entry except one per - // real key, which leaves orphan domain-values entries at steps above - // the unwind target (observed on mainnet for the commitment domain). for domain := range sd.unwindChangesetRaw { sort.Slice(sd.unwindChangesetRaw[domain], func(i, j int) bool { return sd.unwindChangesetRaw[domain][i].Key < sd.unwindChangesetRaw[domain][j].Key @@ -777,7 +818,6 @@ func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - if err := sd.flushDiffSet(ctx, tx); err != nil { return err } @@ -790,6 +830,12 @@ func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { return nil } +func (sd *TemporalMemBatch) Flush(ctx context.Context, tx kv.RwTx) error { + sd.latestStateLock.Lock() + defer sd.latestStateLock.Unlock() + return sd.flushLocked(ctx, tx) +} + func (sd *TemporalMemBatch) flushDiffSet(_ context.Context, tx kv.RwTx) error { for key, changeSet := range sd.pastChangesAccumulator { blockNum := binary.BigEndian.Uint64(common.ToBytesZeroCopy(key[:8])) From 8bad508fac46ad43b646807d538541509652bb41 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 12:53:55 +0000 Subject: [PATCH 048/120] commitment, state/execctx: BFS / budget-driven trunk preload Replaces the depth-first + branch-count-cap enumeration with a breadth-first walk bounded by a RAM byte budget. Depth becomes emergent from (trie shape, contract sparsity, budget) rather than configured directly. Why BFS. With DFS+cap, the budget could be exhausted on one subtree's deep children before the shallower sibling-trunk branches got pinned, producing a "deep narrow slice" pin set that misses the most-shared branches. The d=69 sweep on the prior implementation showed this: pin_count tripled but pin_hit dropped below d=67 because depth-first descent spent the cap on low-reuse leaves. BFS guarantees layer N completes before layer N+1 starts, so the budget is always spent on the highest-value (most-shared) branches first. Once a layer saturates, the remainder spills naturally into the next-most-valuable branches in the next layer. Why a RAM byte budget instead of branch-count cap. Branches vary in encoded size; a fixed branch-count cap can over- or under-shoot the intended RAM cost. A byte budget gives operators a single knob with direct memory semantics, and lets the depth reached vary per-contract based on each trie's shape. Validated on the SSTORE-bloat workload (test_sstore_bloated[10GB-Osaka -NO_CACHE-30M], cold-cgroup 32 GiB / 6c, contract 8eec99...aee7c): budget | pin_count | pin_hit | files_comm | TEST took ---------|-----------|---------|------------|---------- DFS d=68 | 69,905 | 42,216 | 16,346 | 1.34 s (prior) BFS 32MB | 44,685 | 33,386 | 18,585 | 1.46 s (budget-limited) BFS 64MB | 89,365 | 42,637 | 16,243 | 1.25 s (-7% vs DFS) BFS @ 64 MiB beats DFS d=68 in TEST took because it covers depth 69 (max_depth_reached=69) without the DFS+cap regression, while still saturating depths 64-68. Divergences=0 across all variants. Caller env: PIN_TRUNK_RAM_BUDGET_MB (default 32) replaces the prior PIN_TRUNK_MAX_DEPTH knob. --- db/state/execctx/domain_shared.go | 20 +++-- execution/commitment/preload.go | 143 ++++++++++++++++-------------- 2 files changed, 89 insertions(+), 74 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8d11a583561..25c53239220 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1234,13 +1234,15 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // Default 67 covers depths 64-67 = ~16+256+4096 ≈ 4.4K branches max - // per contract for a fully-saturated subtree. Override via - // PIN_TRUNK_MAX_DEPTH for sweep experiments. - maxDepth := 67 - if v := dbg.EnvInt("PIN_TRUNK_MAX_DEPTH", 67); v >= 64 { - maxDepth = v - } + // RAM budget for the per-contract pin tier. Default 32 MiB covers + // a fully-saturated d=68 trunk (~70K branches × ~200 B) with + // headroom; depth reached is emergent from trie shape + budget. + // Override via PIN_TRUNK_RAM_BUDGET_MB for sweep experiments. + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 32) + if ramBudgetMB <= 0 { + ramBudgetMB = 32 + } + ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) var hashes [][]byte for _, hexStr := range strings.Split(pinList, ",") { @@ -1256,7 +1258,7 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } hashes = append(hashes, h) } - logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "max_depth", maxDepth) + logger.Info("[trunk-preload] hashes parsed", "count", len(hashes), "ram_budget_mb", ramBudgetMB) if len(hashes) == 0 { return } @@ -1282,7 +1284,7 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach for i, h := range hashes { started := time.Now() logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, maxDepth, reader, branchCache, logger) + n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index c996943318c..3ade13049b3 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -22,35 +22,42 @@ import ( // the reader in at preload time without dragging in db/state imports. type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) -// PreloadContractTrunk pins commitment branches for the given contract -// from depth 64 (storage subtree root) down to maxDepth into the supplied -// BranchCache. Walks the trie structure depth-by-depth: reads the branch, -// pins it, decodes its child bitmap, and recurses only into children that -// actually exist. +// estimatedEntryOverheadBytes accounts for the per-entry RAM cost +// beyond the encoded value itself: the branchCacheEntry struct +// (~80 B), maphash slot + hash (~40 B), prefix slice (~24 B header + +// content), value slice header (~24 B). Used to predict whether +// pinning the next entry would exceed the configured budget. +const estimatedEntryOverheadBytes = 168 + +// PreloadContractTrunk pins commitment branches for the given contract's +// storage subtree into the supplied BranchCache, breadth-first from +// depth 64 (storage subtree root) outward. The descent is bounded by +// ramBudgetBytes — pinning stops the moment the next entry would push +// estimated total RAM over the budget. The depth reached is therefore +// emergent from (trie shape, contract sparsity, budget), not configured +// directly. // -// contractHash is keccak256(address) — 32 bytes. The trunk lives at the -// commitment-domain prefix that corresponds to nibbles 0..63 of the -// storage path (the contract's account hash); enumeration extends from -// there into nibbles 64..maxDepth. +// Why BFS: a depth-first descent under a hard branch-count cap can +// exhaust budget on one subtree's deep children before completing the +// shallower trunk for sibling branches, producing a pin set that is +// "deep narrow slice" rather than "saturated trunk". Trunk reuse is +// proportional to depth shallowness — each branch at depth N is shared +// across all of its descendant slot reads. BFS guarantees layer N is +// completely pinned before layer N+1 starts, so the budget is always +// spent on the highest-value (most-shared) branches first. // -// Read amplification: total reads = number of branches actually present -// in the trunk (bounded by the contract's storage trie shape). For a -// dense storage subtree, this is ~16+256+4096 ≈ 4.4 K reads at depth 65, -// 66, 67. Sparse subtrees produce fewer. maxDepth caps the descent. +// contractHash is keccak256(address) — 32 bytes. The trunk lives at +// the commitment-domain prefix that corresponds to nibbles 0..63 of +// the storage path; enumeration extends from there into nibbles 64+ +// until the budget is exhausted or the contract's storage trie has +// no further branches. // // Returns the number of pinned branches and any error from the reader. // On error the partial pin set remains in place — those entries are -// still valid cache hits, they just won't see further enumeration below -// the failure point. -// -// Loading strategy: this is the simplest correct implementation — -// recursive enumerate via per-prefix reader lookups. A bulk variant -// using seg.Getter range-scan over sorted .kv would amortize disk -// seeks but requires building a prefix-range API on top of recsplit. -// Defer that optimization until per-prefix lookup is shown to bottleneck. +// still valid cache hits. func PreloadContractTrunk( contractHash []byte, - maxDepth int, + ramBudgetBytes int, reader CommitmentReader, cache *BranchCache, logger log.Logger, @@ -58,84 +65,90 @@ func PreloadContractTrunk( if len(contractHash) != 32 { return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) } - if maxDepth < 64 { - return 0, fmt.Errorf("PreloadContractTrunk: maxDepth %d < 64 (storage subtree starts at depth 64)", maxDepth) - } if cache == nil { return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") } + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } - // Convert contract hash bytes to 64 nibbles. + // Convert contract hash bytes to 64 nibbles — the path prefix + // that anchors all of the contract's storage subtree branches. contractNibbles := make([]byte, 64) for i, b := range contractHash { contractNibbles[2*i] = b >> 4 contractNibbles[2*i+1] = b & 0x0f } - // Hard cap to bound preload work. A fully-saturated 4-level subtree - // is 1+16+256+4096 = 4369 branches; this gives headroom but still - // fails fast on malformed input (e.g. a contract with truly - // pathological branching). Tune downward if this turns out to bite. - const maxBranches = 200000 + type pathDepth struct { + path []byte + depth int + } + queue := []pathDepth{{path: contractNibbles, depth: 64}} pinned := 0 - var stopped bool - var enumerate func(pathNibbles []byte, depth int) error - enumerate = func(pathNibbles []byte, depth int) error { - if stopped { - return nil - } - if pinned >= maxBranches { - stopped = true - if logger != nil { - logger.Warn("[trunk-preload] hit max-branches cap", - "cap", maxBranches, "depth", depth) - } - return nil - } - prefix := nibbles.HexToCompact(pathNibbles) + usedBytes := 0 + maxDepthReached := 64 + budgetExhausted := false + + for len(queue) > 0 { + head := queue[0] + queue = queue[1:] + + prefix := nibbles.HexToCompact(head.path) v, step, found, err := reader(prefix) if err != nil { - return fmt.Errorf("preload at depth %d: %w", depth, err) + return pinned, fmt.Errorf("preload at depth %d: %w", head.depth, err) } if !found { - return nil + continue + } + + entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) + if usedBytes+entryCost > ramBudgetBytes { + budgetExhausted = true + break } + cache.PinEntry(prefix, v, step, "preload-trunk") + usedBytes += entryCost pinned++ - // Periodic progress log so a slow preload is observable. - if logger != nil && pinned%500 == 0 { - logger.Info("[trunk-preload] progress", "pinned", pinned, "depth", depth) + if head.depth > maxDepthReached { + maxDepthReached = head.depth } - if depth >= maxDepth { - return nil + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload] progress", + "pinned", pinned, "depth", head.depth, + "used_mb", usedBytes/(1<<20)) } + + // Decode bitmap and queue children. Branch encoding: + // 2-byte touchMap || 2-byte bitmap || per-child data. Only + // queue children that actually exist in the trie (bit set). if len(v) < 4 { - return nil + continue } - // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. - // Only recurse into children that actually exist (bit set in bitmap). bitmap := binary.BigEndian.Uint16(v[2:4]) for n := 0; n < 16; n++ { if bitmap&(1< Date: Sun, 10 May 2026 13:07:05 +0000 Subject: [PATCH 049/120] =?UTF-8?q?state/execctx:=20raise=20PIN=5FTRUNK=5F?= =?UTF-8?q?RAM=5FBUDGET=5FMB=20default=2032=20=E2=86=92=2064?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empirical sweep (32 / 64 / 256 MiB) on the SSTORE-bloat workload: budget | TEST took | mgas/s | files_comm ---------|-----------|--------|------------ 32 MiB | 1.46 s | 20.5 | 18,585 (under-coverage) 64 MiB | 1.25 s | 24.0 | 16,243 (perf knee) 256 MiB | 1.27 s | 23.7 | 15,542 (diminishing) 64 MiB covers the structural d=68 saturation (~70K branches) plus the most-shared d=69 branches. Going past 64 MiB delivers only −2 ms TEST took for +35 s preload — net negative. 32 MiB defaulted previously because of an inaccurate per-entry-bytes estimate (real cost ~720 B vs assumed ~200 B). Reset to the validated knee. --- db/state/execctx/domain_shared.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 25c53239220..859d81319c0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1234,13 +1234,15 @@ func (sd *SharedDomains) touchChangedKeys(tx kv.TemporalTx, d kv.Domain, fromTxN // Own-tx async sidesteps both. Caller must guarantee fire-once via // BranchCache.TryClaimPreload (the only call site does this). func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCache, db kv.TemporalRoDB, pinList string, logger log.Logger) { - // RAM budget for the per-contract pin tier. Default 32 MiB covers - // a fully-saturated d=68 trunk (~70K branches × ~200 B) with - // headroom; depth reached is emergent from trie shape + budget. - // Override via PIN_TRUNK_RAM_BUDGET_MB for sweep experiments. - ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 32) + // RAM budget per contract. 64 MiB is the perf knee on the + // SSTORE-bloat workload — covers the structural d=68 saturation + // (~70K branches) plus most-shared d=69 branches, beyond which + // marginal returns drop sharply (256 MiB sweep showed −2 ms TEST + // took for +35 s preload). Override via PIN_TRUNK_RAM_BUDGET_MB + // for tuning on different workload classes. + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 64) if ramBudgetMB <= 0 { - ramBudgetMB = 32 + ramBudgetMB = 64 } ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) From d494eec8fe249a1abcb5dfa95d3bad105abcd4aa Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:21:46 +0000 Subject: [PATCH 050/120] state/execctx: clamp PIN_TRUNK_RAM_BUDGET_MB at 64 MiB per contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hard-cap the per-contract pin budget at 64 MiB. Larger values silently clamp with a warning log so an operator can see their override was ignored. Why 64 MiB max for this PR. Larger budgets are not safe to ship until two prerequisite levers exist: 1. Per-account touch-driven minimisation (R11) — without this, deeper pin sets accumulate low-reuse branches that consume RAM without delivering perf. Today's 256 MiB sweep showed 19.9 % hit-rate-per-pin vs 47.7 % at 64 MiB. 2. Global RAM cap aware of available memory (R12) — without this, an operator promoting many contracts at high per-contract budgets could pin GBs of state on a small host. Override is plausibly justified for narrow cases (a measured-hot mainnet contract whose perf curve genuinely needs deeper coverage, operator diagnostic profiling) but those paths land once R11 + R12 + a per-contract whitelist exist (R13). --- db/state/execctx/domain_shared.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 859d81319c0..8fbf18252e0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1238,11 +1238,31 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach // SSTORE-bloat workload — covers the structural d=68 saturation // (~70K branches) plus most-shared d=69 branches, beyond which // marginal returns drop sharply (256 MiB sweep showed −2 ms TEST - // took for +35 s preload). Override via PIN_TRUNK_RAM_BUDGET_MB - // for tuning on different workload classes. - ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", 64) + // took for +35 s preload). + // + // 64 MiB is the per-contract MAX for this PR. Larger budgets are + // not safe to ship without (a) per-account touch-driven minimisation + // that prunes the pinned set to the actually-useful subset (R11) + // and (b) a global RAM cap aware of available memory (R12). Until + // both land, an operator setting 256 MiB across many promoted + // contracts could pin GBs without material perf benefit. + // + // Override is plausibly justified in narrow cases — a measured-hot + // mainnet contract whose perf curve genuinely requires deeper + // coverage, or operator-driven diagnostic profiling. NOT in scope + // for this PR; an override path lands once R11 + R12 + a + // per-contract whitelist mechanism exist. Until then, larger values + // silently clamp with a warning so an operator can see they were + // ignored. + const maxPerContractBudgetMB = 64 + ramBudgetMB := dbg.EnvInt("PIN_TRUNK_RAM_BUDGET_MB", maxPerContractBudgetMB) if ramBudgetMB <= 0 { - ramBudgetMB = 64 + ramBudgetMB = maxPerContractBudgetMB + } + if ramBudgetMB > maxPerContractBudgetMB { + logger.Warn("[trunk-preload] clamping budget to per-contract max", + "requested_mb", ramBudgetMB, "clamped_mb", maxPerContractBudgetMB) + ramBudgetMB = maxPerContractBudgetMB } ramBudgetBytes := ramBudgetMB << 20 logger.Info("[trunk-preload] entering", "pin_list_raw", pinList) From 6ea7ff8bb5f8aa4340d2f50c9c63b9ad6fe9cb27 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:34:26 +0000 Subject: [PATCH 051/120] commitment/branch_cache: pin-aware Invalidate, GetWithOrigin, Clear Three accessor functions previously skipped the pinned tier, leaving correctness gaps that didn't bite the bench (which doesn't exercise unwind / fork-validation paths against pinned entries) but would manifest in production: - Invalidate(prefix) only deleted from root + LRU tail. A canonical store update for a pinned prefix would leave stale bytes pinned forever. Now deletes from pinned tier too. - GetWithOrigin(prefix) returned (nil,...,false) for pinned entries. The divergence-detection probe used this and would miss real cache-vs-canonical disagreements on pinned branches. - Clear() left the pinned tier intact and didn't reset pinned-tier counters. Fork-validation reset paths could carry stale pinned bytes across trie roots. Lifecycle context: pinned entries normally refresh in-place via Put on SD.Flush (pin survives every-block writes). Invalidate is the escape hatch for events where pinned bytes must be discarded outright (unwind, fork-validation reset, manual demotion). Clear is the full reset, sized for fork-validation paths that need a clean slate. --- execution/commitment/branch_cache.go | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 1f138504a11..f0b4970327c 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -446,6 +446,8 @@ func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, var entry *branchCacheEntry if isRootPrefix(prefix) { entry = c.root.Load() + } else if pinnedEntry, pinnedOk := c.pinned.Get(prefix); pinnedOk { + entry = pinnedEntry } else { entry, ok = c.tail.Get(prefix) if !ok { @@ -470,26 +472,39 @@ func (c *BranchCache) MarkDirty(prefix []byte) { } } -// Invalidate removes the entry at prefix entirely. Use when the caller -// knows the canonical store has changed and the cached entry should not -// be served at all (vs MarkDirty which keeps the entry but blocks -// PutIfClean overwrites). +// Invalidate removes the entry at prefix entirely from whichever tier +// holds it. Use when the caller knows the canonical store has changed +// and the cached entry should not be served at all (vs MarkDirty which +// keeps the entry but blocks PutIfClean overwrites). +// +// Pinned-tier note: Invalidate does delete from the pinned tier. The +// usual lifecycle for pinned entries is in-place refresh via Put on +// SD.Flush (so pin survives every-block writes); Invalidate is the +// escape hatch for events where the pinned bytes must be discarded +// outright (unwind, fork-validation reset, manual demotion). func (c *BranchCache) Invalidate(prefix []byte) { if isRootPrefix(prefix) { c.root.Store(nil) return } + c.pinned.Delete(prefix) c.tail.Delete(prefix) } -// Clear empties the cache and resets stats counters. Use on Reset / -// fork-validation paths to ensure stale entries from one trie root are -// not served against a different root. +// Clear empties the cache and resets stats counters across ALL tiers +// (root, pinned, LRU tail). Use on Reset / fork-validation paths to +// ensure stale entries from one trie root are not served against a +// different root. After Clear, the pinned tier is empty; callers +// using trunk preload need to re-issue PreloadContractTrunk to +// repopulate it. func (c *BranchCache) Clear() { c.root.Store(nil) + c.pinned = maphash.NewMap[*branchCacheEntry]() c.tail.Purge() c.rootHits.Store(0) c.rootMisses.Store(0) + c.pinnedHits.Store(0) + c.pinnedMisses.Store(0) c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) From 9ef3b9c6a023443f9b0fcfa0fa114ead6478f7d5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:37:30 +0000 Subject: [PATCH 052/120] state/execctx: invalidate stale BranchCache entries on SD.Unwind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After unwind the canonical commitment-domain values are rolled back; any cached entries for keys touched in the unwind window now hold stale bytes vs the post-unwind canonical state. Without this fix, a read after unwind that hits the cache before the next Put-on-Flush refresh would return the pre-unwind value — wrong-trie-root waiting to happen the next time the relevant key is touched. Selective per-key invalidation rather than Clear() — only the commitment-domain keys actually in the changeset get dropped. Pinned trunk branches whose keys weren't in the changeset stay warm; LRU tail entries unrelated to the unwind survive. Clear() remains the escape hatch for full reset (fork-validation paths that need a clean slate). Bench doesn't currently exercise unwind so this is a defensive fix ahead of cross-block stress testing. Pin-aware Invalidate (committed previously) is what makes this safe — without it, the per-key delete would leave pinned entries holding stale bytes. --- db/state/execctx/domain_shared.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8fbf18252e0..24c4bd5030b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -539,6 +539,16 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) + // After unwind the canonical commitment-domain values are rolled + // back; any cached entries for keys touched in the unwind window + // now hold stale bytes vs the post-unwind canonical state. Drop + // those specifically — the rest of the cache (including pinned + // branches whose keys weren't in the changeset) stays warm. + if sd.branchCache != nil && changeset != nil { + for _, diff := range changeset[kv.CommitmentDomain] { + sd.branchCache.Invalidate([]byte(diff.Key)) + } + } } func (sd *SharedDomains) Trace() bool { From 878f04a1b16f1615634dc24e40a07153b2df6c6e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:55:34 +0000 Subject: [PATCH 053/120] commitment/preload: split BFS into resumable ContractTrunkPreload Refactor the one-shot PreloadContractTrunk into a stateful ContractTrunkPreload that supports incremental BFS via Run, plus a backward-compatible PreloadContractTrunk wrapper. Why split. The phased adaptive load (initial sync view at promotion + per-block extension while contract stays hot) needs to pause and resume the BFS between calls. The previous one-shot signature required burst-loading the full budget upfront, which paid the ~28 s cold-disk preload cost at every promotion regardless of whether the contract turned out to be worth it. The new shape: state := NewContractTrunkPreload(contractHash) // Initial view: small budget for d=64-67 trunk state.Run(initialBudget, reader, cache, logger) // Per-block extension while contract stays hot: state.Run(perBlockBudget, reader, cache, logger) // Repeat until queue empty or contract demoted. PreloadContractTrunk is now a wrapper around NewContractTrunkPreload + a single Run, preserving existing call sites (PIN_CONTRACT_TRUNKS env hook in domain_shared.go) without changes. Behavior-preserving refactor. Validated: BFS 64 MiB SSTORE-bloat bench reproduces baseline byte-for-byte (pin_count=89365, pin_hit=42637, files_comm=16244, took=1.22 s, divergences=0). --- execution/commitment/preload.go | 194 ++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 72 deletions(-) diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 3ade13049b3..e46bba6d825 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -16,7 +16,7 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// CommitmentReader is the read interface PreloadContractTrunk needs: +// CommitmentReader is the read interface ContractTrunkPreload needs: // GetLatest on the CommitmentDomain. Returns (value, step, found, err). // Decoupled from any specific tx/aggregator type so callers can plug // the reader in at preload time without dragging in db/state imports. @@ -29,97 +29,107 @@ type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, er // pinning the next entry would exceed the configured budget. const estimatedEntryOverheadBytes = 168 -// PreloadContractTrunk pins commitment branches for the given contract's -// storage subtree into the supplied BranchCache, breadth-first from -// depth 64 (storage subtree root) outward. The descent is bounded by -// ramBudgetBytes — pinning stops the moment the next entry would push -// estimated total RAM over the budget. The depth reached is therefore -// emergent from (trie shape, contract sparsity, budget), not configured -// directly. -// -// Why BFS: a depth-first descent under a hard branch-count cap can -// exhaust budget on one subtree's deep children before completing the -// shallower trunk for sibling branches, producing a pin set that is -// "deep narrow slice" rather than "saturated trunk". Trunk reuse is -// proportional to depth shallowness — each branch at depth N is shared -// across all of its descendant slot reads. BFS guarantees layer N is -// completely pinned before layer N+1 starts, so the budget is always -// spent on the highest-value (most-shared) branches first. -// -// contractHash is keccak256(address) — 32 bytes. The trunk lives at -// the commitment-domain prefix that corresponds to nibbles 0..63 of -// the storage path; enumeration extends from there into nibbles 64+ -// until the budget is exhausted or the contract's storage trie has -// no further branches. +// pathDepth pairs a nibble path with the trie depth it terminates at. +// Used as the BFS queue element. +type pathDepth struct { + path []byte + depth int +} + +// ContractTrunkPreload holds the resumable state of a BFS preload for +// one contract's storage subtree. Created by NewContractTrunkPreload +// and advanced incrementally via Run; this lets callers split the +// preload into an initial sync view (small budget for d=64-67) and +// per-block extensions (additional budget while the contract stays +// hot), instead of burst-loading the full budget at promotion time. // -// Returns the number of pinned branches and any error from the reader. -// On error the partial pin set remains in place — those entries are -// still valid cache hits. -func PreloadContractTrunk( - contractHash []byte, - ramBudgetBytes int, - reader CommitmentReader, - cache *BranchCache, - logger log.Logger, -) (int, error) { +// Caller MUST NOT use the same instance from multiple goroutines. +type ContractTrunkPreload struct { + contractHash []byte + queue []pathDepth + pinned int + usedBytes int + maxDepthReached int +} + +// NewContractTrunkPreload constructs a preload state seeded at depth 64 +// (storage subtree root) for the given contract. The contract's +// storage subtree path is keccak256(address) — 32 bytes / 64 nibbles. +func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) { if len(contractHash) != 32 { - return 0, fmt.Errorf("PreloadContractTrunk: contractHash must be 32 bytes, got %d", len(contractHash)) - } - if cache == nil { - return 0, fmt.Errorf("PreloadContractTrunk: cache is nil") + return nil, fmt.Errorf("NewContractTrunkPreload: contractHash must be 32 bytes, got %d", len(contractHash)) } - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - - // Convert contract hash bytes to 64 nibbles — the path prefix - // that anchors all of the contract's storage subtree branches. contractNibbles := make([]byte, 64) for i, b := range contractHash { contractNibbles[2*i] = b >> 4 contractNibbles[2*i+1] = b & 0x0f } + return &ContractTrunkPreload{ + contractHash: contractHash, + queue: []pathDepth{{path: contractNibbles, depth: 64}}, + maxDepthReached: 64, + }, nil +} - type pathDepth struct { - path []byte - depth int +// Run advances the BFS preload, pinning branches into cache until +// either the additional budget is exhausted or the queue is empty. +// Returns the number of entries pinned during THIS call (not the +// cumulative total — see PinnedTotal), whether the queue is now +// empty (preload fully complete), and any error from the reader. +// +// Calling Run repeatedly with small budgets implements the phased +// load: an initial Run with the InitialBudget covers depths 64-67; +// each subsequent Run extends BFS one chunk further. Per-block +// callers should size the chunk to fit within inter-block idle time. +// +// On reader error the partial pin set survives; the queue position +// is preserved so a retry on the next Run picks up where it left off. +func (p *ContractTrunkPreload) Run( + additionalBudgetBytes int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (newlyPinned int, queueEmpty bool, err error) { + if cache == nil { + return 0, false, fmt.Errorf("ContractTrunkPreload.Run: cache is nil") + } + if additionalBudgetBytes <= 0 { + return 0, len(p.queue) == 0, nil } - queue := []pathDepth{{path: contractNibbles, depth: 64}} - pinned := 0 - usedBytes := 0 - maxDepthReached := 64 - budgetExhausted := false + chunkUsedBytes := 0 + chunkPinned := 0 - for len(queue) > 0 { - head := queue[0] - queue = queue[1:] + for len(p.queue) > 0 { + head := p.queue[0] + p.queue = p.queue[1:] prefix := nibbles.HexToCompact(head.path) - v, step, found, err := reader(prefix) - if err != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", head.depth, err) + v, step, found, rerr := reader(prefix) + if rerr != nil { + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", head.depth, rerr) } if !found { continue } entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) - if usedBytes+entryCost > ramBudgetBytes { - budgetExhausted = true + if chunkUsedBytes+entryCost > additionalBudgetBytes { + // Re-queue the head so a future Run picks it up. + p.queue = append([]pathDepth{head}, p.queue...) break } cache.PinEntry(prefix, v, step, "preload-trunk") - usedBytes += entryCost - pinned++ - if head.depth > maxDepthReached { - maxDepthReached = head.depth + chunkUsedBytes += entryCost + chunkPinned++ + if head.depth > p.maxDepthReached { + p.maxDepthReached = head.depth } - if logger != nil && pinned%5000 == 0 { + if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { logger.Info("[trunk-preload] progress", - "pinned", pinned, "depth", head.depth, - "used_mb", usedBytes/(1<<20)) + "pinned", p.pinned+chunkPinned, "depth", head.depth, + "used_mb", (p.usedBytes+chunkUsedBytes)/(1<<20)) } // Decode bitmap and queue children. Branch encoding: @@ -136,20 +146,60 @@ func PreloadContractTrunk( childPath := make([]byte, len(head.path)+1) copy(childPath, head.path) childPath[len(head.path)] = byte(n) - queue = append(queue, pathDepth{path: childPath, depth: head.depth + 1}) + p.queue = append(p.queue, pathDepth{path: childPath, depth: head.depth + 1}) } } + p.pinned += chunkPinned + p.usedBytes += chunkUsedBytes + return chunkPinned, len(p.queue) == 0, nil +} + +// PinnedTotal returns the cumulative number of branches pinned by +// this preload state across all Run calls. +func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } + +// UsedBytes returns the cumulative byte cost of this preload's pin set. +func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } + +// QueueRemaining returns the number of paths still queued for descent. +// Zero means the preload is complete; further Run calls are no-ops. +func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } + +// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } + +// PreloadContractTrunk runs the full BFS preload for a contract in a +// single call, bounded by ramBudgetBytes. Convenience wrapper around +// NewContractTrunkPreload + Run for callers that don't need phased +// loading. Existing one-shot trigger paths (PIN_CONTRACT_TRUNKS env +// hook) use this; the adaptive layer holds a *ContractTrunkPreload +// and calls Run incrementally. +func PreloadContractTrunk( + contractHash []byte, + ramBudgetBytes int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (int, error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreload(contractHash) + if err != nil { + return 0, err + } + pinned, queueEmpty, err := p.Run(ramBudgetBytes, reader, cache, logger) if logger != nil { logger.Info("[trunk-preload] complete", "contract_hash", fmt.Sprintf("%x", contractHash), "ram_budget_mb", ramBudgetBytes/(1<<20), - "used_mb", usedBytes/(1<<20), + "used_mb", p.UsedBytes()/(1<<20), "pinned", pinned, - "max_depth_reached", maxDepthReached, - "budget_exhausted", budgetExhausted, - "queue_remaining", len(queue), + "max_depth_reached", p.MaxDepthReached(), + "budget_exhausted", !queueEmpty, + "queue_remaining", p.QueueRemaining(), "cache_pinned_total", cache.PinnedCount()) } - return pinned, nil + return pinned, err } From a2b6ab508ca7a8eb014895f56ba32eac86fae363 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 13:58:06 +0000 Subject: [PATCH 054/120] commitment/branch_cache: add miss-callback hook for adaptive controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SetMissCallback / MissCallback type and ContractHashFromPrefix helper. The callback fires whenever lookup misses ALL three tiers (root, pinned, LRU tail) — the deepest signal the cache has that a file read will follow. Hot path stays cheap: the no-callback case is a single atomic.Pointer load and a nil check. Implementations must be lock-free. ContractHashFromPrefix decodes the contract's 32-byte storage-subtree prefix (1-byte HP flag + 64 nibbles packed). Returns false for prefixes shorter than 33 bytes (account-trie branches), so the adaptive controller naturally filters out non-storage-trunk misses. The callback alone does no policy — the AdaptivePinController (commit 24c) registers a callback that attributes misses per contract and decides promotions. Cache stays simple; controller owns the policy. --- execution/commitment/branch_cache.go | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index f0b4970327c..d7c22d2e20d 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -192,6 +192,14 @@ type BranchCache struct { // stale bytes. writeSeq atomic.Uint64 + // onMiss is an optional hook fired when lookup misses all three + // tiers (root, pinned, LRU). Used by the adaptive trunk-pin + // controller to attribute miss pressure per-contract and decide + // promotions. Stored as atomic.Pointer so registration is + // lock-free and the hot read path skips the dereference cleanly + // when no callback is installed. + onMiss atomic.Pointer[MissCallback] + // preloadClaimed is set the first time TryClaimPreload is called. // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in // SharedDomains construction) so the preload goroutine fires @@ -292,6 +300,7 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { entry := c.root.Load() if entry == nil { c.rootMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.rootHits.Add(1) @@ -307,12 +316,21 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { entry, ok := c.tail.Get(prefix) if !ok { c.tailMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.tailHits.Add(1) return entry, true } +// fireOnMiss invokes the registered miss callback (if any). Hot path — +// the no-callback case is a single atomic load and a nil check. +func (c *BranchCache) fireOnMiss(prefix []byte) { + if cb := c.onMiss.Load(); cb != nil { + (*cb)(prefix) + } +} + func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { if isRootPrefix(prefix) { c.root.Store(entry) @@ -355,6 +373,40 @@ func (c *BranchCache) PinnedCount() int { return c.pinned.Len() } +// MissCallback is invoked when lookup misses ALL three tiers (root, +// pinned, LRU tail). Called on the hot read path; implementations +// must be lock-free or short and not block. +type MissCallback func(prefix []byte) + +// SetMissCallback installs a hook fired on every triple-miss. Pass +// nil to clear. Used by the adaptive controller to attribute miss +// pressure per contract; the cache itself does no per-contract +// bookkeeping. Replaces any prior callback atomically. +func (c *BranchCache) SetMissCallback(cb MissCallback) { + if cb == nil { + c.onMiss.Store(nil) + return + } + c.onMiss.Store(&cb) +} + +// ContractHashFromPrefix extracts the 32-byte contract hash from a +// commitment-trunk prefix. Returns (hash, true) if the prefix is a +// storage-trunk prefix (depth >= 64, encoded as compact-hex with the +// contract's 64-nibble keccak prefix), or (_, false) for shorter +// prefixes (account-trie branches at depth < 64). +// +// Compact-hex encoding: 1-byte HP flag, then nibble-pairs packed +// 2-per-byte. The contract's 64-nibble path occupies 32 bytes after +// the flag byte, so a storage-trunk prefix is at least 33 bytes. +func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { + if len(prefix) < 33 { + return hash, false + } + copy(hash[:], prefix[1:33]) + return hash, true +} + // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). From be2616e6c2653ba1a44b85ba97df04f732ff6a78 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:01:35 +0000 Subject: [PATCH 055/120] commitment: AdaptivePinController for promotion / extension / demotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the policy layer that decides which contracts to pin based on observed miss pressure. Uses the cache miss-callback (24b) to attribute per-contract pressure and the resumable ContractTrunkPreload (24a) to phase the load: - Promotion: a contract whose cache misses cross PromoteThresholdMisses in a single block gets a synchronous initial-view preload (4 MiB default, ~1.4 s, covers d=64-67 trunk). Bounded by MaxPromotedContracts (default 8 → 512 MiB total max RAM). - Extension: each block where a promoted contract had any miss, the controller advances BFS by ExtensionBudgetBytes (8 MiB default). Stops at PerContractMaxBudgetBytes (64 MiB ceiling). - Demotion: after DemoteCooldownBlocks (5 default) consecutive cold blocks, the controller invalidates the contract's pin set. To support this, ContractTrunkPreload now tracks PinnedPrefixes — the slice of prefixes added to the cache, copied on each PinEntry so demotion has stable handles. Defaults are conservative — designed for safe default-on shipping with no per-host tuning. R11 (touch-driven minimisation) and R12 (global memory cap aware of available RAM) are future work that will replace the static MaxPromotedContracts × PerContractMaxBudgetBytes calculation with adaptive policy. The controller itself is decoupled from wiring: NewAdaptivePinController constructs it, Bind installs the cache callback, OnBlockComplete is called by the host (commit 24d wires this into SD construction). --- execution/commitment/adaptive_pin.go | 333 +++++++++++++++++++++++++++ execution/commitment/preload.go | 19 ++ 2 files changed, 352 insertions(+) create mode 100644 execution/commitment/adaptive_pin.go diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go new file mode 100644 index 00000000000..9a21ce1fdde --- /dev/null +++ b/execution/commitment/adaptive_pin.go @@ -0,0 +1,333 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "context" + "encoding/hex" + "sync" + "sync/atomic" + + "github.com/erigontech/erigon/common/log/v3" +) + +// AdaptivePinControllerConfig sets the policy knobs for the adaptive +// trunk-pin controller. Defaults are tuned for the SSTORE-bloat +// workload class (single contract dominating storage reads); other +// workload classes may want different values. +type AdaptivePinControllerConfig struct { + // PromoteThresholdMisses — minimum cache misses (per block) for a + // contract to be promoted. Default 100. + PromoteThresholdMisses uint64 + + // MaxPromotedContracts — hard cap on simultaneously-pinned + // contracts. Bounds total pin RAM = MaxPromotedContracts × + // PerContractMaxBudgetBytes. Default 8 (× 64 MiB = 512 MiB max). + MaxPromotedContracts int + + // DemoteCooldownBlocks — number of consecutive blocks with zero + // misses for a promoted contract before it gets demoted (and its + // pin set invalidated). Default 5. + DemoteCooldownBlocks int + + // InitialViewBudgetBytes — RAM budget for the synchronous + // initial-view preload at promotion. Default 4 MiB (covers + // d=64-67 with headroom; ~1.4 s on cold disk). + InitialViewBudgetBytes int + + // ExtensionBudgetBytes — RAM budget for the per-block extension + // step on already-promoted contracts. Default 8 MiB (~25 K + // branches per block at typical entry cost). + ExtensionBudgetBytes int + + // PerContractMaxBudgetBytes — hard ceiling on the cumulative pin + // budget per contract. Extensions stop when this is reached even + // if the BFS queue isn't empty. Default 64 MiB (matches the + // per-contract max enforced in the env hook). + PerContractMaxBudgetBytes int +} + +// DefaultAdaptivePinControllerConfig returns the production defaults. +// Conservative across the board so default-on shipping doesn't pin +// memory speculatively. +func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { + return AdaptivePinControllerConfig{ + PromoteThresholdMisses: 100, + MaxPromotedContracts: 8, + DemoteCooldownBlocks: 5, + InitialViewBudgetBytes: 4 << 20, + ExtensionBudgetBytes: 8 << 20, + PerContractMaxBudgetBytes: 64 << 20, + } +} + +// AdaptivePinController watches per-contract miss pressure on a +// BranchCache and decides which contracts to pin (with a sync initial +// view) and grow (per-block extension) or demote (invalidate the pin +// set after sustained inactivity). +// +// Lifecycle: +// - Construct with NewAdaptivePinController, wire to a cache via Bind +// - On every triple-miss, the cache calls back; the controller +// records a per-contract miss count +// - The host (SD or stage loop) calls OnBlockComplete at block +// boundaries with a CommitmentReader factory; the controller +// then promotes / extends / demotes based on the per-block +// miss snapshot +type AdaptivePinController struct { + cache *BranchCache + cfg AdaptivePinControllerConfig + logger log.Logger + + // misses holds per-contract miss counts since last OnBlockComplete. + // sync.Map is appropriate: writes are frequent (every triple-miss), + // reads happen once per block boundary. + misses sync.Map // [32]byte → *atomic.Uint64 + + // states holds per-promoted-contract bookkeeping. Protected by mu. + mu sync.Mutex + states map[[32]byte]*adaptiveContractState +} + +type adaptiveContractState struct { + contractHash [32]byte + promotedAtBlock uint64 + preload *ContractTrunkPreload + coldBlocksInARow int +} + +// NewAdaptivePinController constructs a controller bound to the given +// cache. Use Bind to install the cache miss-callback (separate so a +// caller can keep a controller for telemetry without wiring it into +// the read path). +func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfig, logger log.Logger) *AdaptivePinController { + if cfg.InitialViewBudgetBytes <= 0 { + cfg.InitialViewBudgetBytes = 4 << 20 + } + if cfg.ExtensionBudgetBytes <= 0 { + cfg.ExtensionBudgetBytes = 8 << 20 + } + if cfg.PerContractMaxBudgetBytes <= 0 { + cfg.PerContractMaxBudgetBytes = 64 << 20 + } + if cfg.MaxPromotedContracts <= 0 { + cfg.MaxPromotedContracts = 8 + } + if cfg.DemoteCooldownBlocks <= 0 { + cfg.DemoteCooldownBlocks = 5 + } + if cfg.PromoteThresholdMisses == 0 { + cfg.PromoteThresholdMisses = 100 + } + return &AdaptivePinController{ + cache: cache, + cfg: cfg, + logger: logger, + states: make(map[[32]byte]*adaptiveContractState), + } +} + +// Bind installs the controller's miss-callback on the cache. After +// Bind, every triple-miss attributable to a storage-trunk prefix +// (length >= 33 B) is counted toward the corresponding contract. +// +// Safe to call multiple times — Bind replaces any prior callback. +// Call SetMissCallback(nil) on the cache directly to unbind. +func (c *AdaptivePinController) Bind() { + c.cache.SetMissCallback(c.onCacheMiss) +} + +func (c *AdaptivePinController) onCacheMiss(prefix []byte) { + hash, ok := ContractHashFromPrefix(prefix) + if !ok { + return + } + v, _ := c.misses.LoadOrStore(hash, new(atomic.Uint64)) + v.(*atomic.Uint64).Add(1) +} + +// OnBlockComplete consumes the per-block miss snapshot and decides +// promotions, extensions, and demotions. Called by the host at block +// boundaries (after SD.Flush). The reader is the CommitmentReader +// for the just-committed state, used by initial-view preload and +// per-block extension. +// +// Synchronous: initial-view preloads run inline so the new pin set +// is available for the NEXT block's reads. Extensions also run +// inline; sized so per-block work fits within typical inter-block +// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). +// +// Logs a [adaptive-pin] line per block when any state changes or +// promoted contracts exist. +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { + misses := c.snapshotMisses() + + c.mu.Lock() + defer c.mu.Unlock() + + var promoted, extended, demoted int + + // Already-promoted contracts: extend on hot, demote on cold. + for hash, state := range c.states { + n, hadMisses := misses[hash] + if hadMisses && n > 0 { + state.coldBlocksInARow = 0 + delete(misses, hash) + // Extend if budget remains and queue not empty. + if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() + step := c.cfg.ExtensionBudgetBytes + if step > remaining { + step = remaining + } + if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) + } else { + extended++ + } + } + continue + } + state.coldBlocksInARow++ + if state.coldBlocksInARow >= c.cfg.DemoteCooldownBlocks { + c.demoteLocked(hash, state) + delete(c.states, hash) + demoted++ + } + } + + // New promotion candidates: contracts whose miss count crossed the + // threshold. Cap at MaxPromotedContracts; pick the highest-miss + // candidates first (greedy, simple, sufficient for v1). + if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { + candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) + for _, hash := range candidates { + p, err := NewContractTrunkPreload(hash[:]) + if err != nil { + continue + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) + // Roll back partial pin so we don't leak entries + // for a contract that won't be in c.states. + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + continue + } + c.states[hash] = &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + preload: p, + } + promoted++ + } + } + + if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { + c.logger.Info("[adaptive-pin]", + "block", blockNum, + "promoted_total", len(c.states), + "promoted_this_block", promoted, + "extended_this_block", extended, + "demoted_this_block", demoted, + "cache_pinned_total", c.cache.PinnedCount()) + } +} + +// snapshotMisses atomically reads + zeros every per-contract miss +// counter. Called once per OnBlockComplete. +func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { + out := make(map[[32]byte]uint64) + c.misses.Range(func(k, v any) bool { + hash := k.([32]byte) + n := v.(*atomic.Uint64).Swap(0) + if n > 0 { + out[hash] = n + } + return true + }) + return out +} + +// demoteLocked invalidates every pinned prefix for the contract. +// Caller must hold c.mu. +func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { + for _, prefix := range state.preload.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + if c.logger != nil { + c.logger.Info("[adaptive-pin] demoted", + "hash", hex.EncodeToString(hash[:]), + "pinned_was", state.preload.PinnedTotal(), + "used_mb_was", state.preload.UsedBytes()/(1<<20), + "cold_blocks", state.coldBlocksInARow) + } +} + +// PromotedContracts returns the hashes of currently-pinned contracts. +// Snapshot — the slice may be stale by the time the caller uses it. +// Used by metrics + debug diagnostics. +func (c *AdaptivePinController) PromotedContracts() [][32]byte { + c.mu.Lock() + defer c.mu.Unlock() + out := make([][32]byte, 0, len(c.states)) + for h := range c.states { + out = append(out, h) + } + return out +} + +// pickPromotionCandidates selects up to maxN contract hashes from +// misses whose count exceeds threshold, preferring highest-miss first. +// Linear scan — adequate for the small N we expect (typically ≤16 +// candidates per block; sort is cheap). +func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { + if maxN <= 0 { + return nil + } + type cand struct { + hash [32]byte + n uint64 + } + var pool []cand + for h, n := range misses { + if n >= threshold { + pool = append(pool, cand{h, n}) + } + } + if len(pool) > maxN { + // Partial sort: O(N×maxN) for small maxN; avoids importing sort. + for i := 0; i < maxN; i++ { + best := i + for j := i + 1; j < len(pool); j++ { + if pool[j].n > pool[best].n { + best = j + } + } + pool[i], pool[best] = pool[best], pool[i] + } + pool = pool[:maxN] + } + out := make([][32]byte, len(pool)) + for i, c := range pool { + out[i] = c.hash + } + return out +} + +func (c *AdaptivePinController) warnf(msg string, kv ...any) { + if c.logger != nil { + c.logger.Warn(msg, kv...) + } +} + +// ctx unused for now; reserved for future cancellation of in-flight +// preloads (R3 bulk preload + cancellable extension). +var _ = context.Background diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index e46bba6d825..88fa7cc2677 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -43,10 +43,14 @@ type pathDepth struct { // per-block extensions (additional budget while the contract stays // hot), instead of burst-loading the full budget at promotion time. // +// Tracks its pinned prefixes so callers can Invalidate the contract's +// pin set on demotion (PinnedPrefixes returns the slice). +// // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte queue []pathDepth + pinnedPrefixes [][]byte pinned int usedBytes int maxDepthReached int @@ -121,6 +125,12 @@ func (p *ContractTrunkPreload) Run( } cache.PinEntry(prefix, v, step, "preload-trunk") + // Copy prefix because nibbles.HexToCompact may return a slice + // over a reused buffer; we need a stable handle for later + // Invalidate on demotion. + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + p.pinnedPrefixes = append(p.pinnedPrefixes, prefixCopy) chunkUsedBytes += entryCost chunkPinned++ if head.depth > p.maxDepthReached { @@ -169,6 +179,15 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } +// PinnedPrefixes returns the prefix slices this preload has added to +// the cache so far. The adaptive controller uses this to Invalidate +// the contract's pin set on demotion. The returned slice aliases +// internal storage; do not mutate. +func (p *ContractTrunkPreload) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } + +// ContractHash returns the contract this preload targets. +func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } + // PreloadContractTrunk runs the full BFS preload for a contract in a // single call, bounded by ramBudgetBytes. Convenience wrapper around // NewContractTrunkPreload + Run for callers that don't need phased From b082c93a4ddbc1cbdd61232120df8a09ea1871f2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:16:38 +0000 Subject: [PATCH 056/120] state/execctx: wire AdaptivePinController into SharedDomains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three integration points: 1. Construction in NewSharedDomainsWithTrieVariant — alongside the branchCache assignment. Inert until Bind. Skipped when DISABLE_ADAPTIVE_PIN env is set (defensive escape hatch). 2. Bind in EnableParaTrieDB — installs the cache miss-callback so the controller starts observing per-contract pressure. Runs AFTER the env-driven PIN_CONTRACT_TRUNKS forced preload, so operator overrides land before the controller starts attribution. 3. OnBlockComplete from SD.Flush — fires after FlushWithCallback has updated the cache for this batch. Reader uses the in-flight Flush tx so initial-view preload of newly-promoted contracts sees the just-committed bytes. Uses sd.txNum as the controller's "tick" — per-batch granularity, so cooldown thresholds map to batches not blocks (tunable via controller config if finer granularity is needed later). Operator override paths preserved: - PIN_CONTRACT_TRUNKS — forces specific contracts to pin regardless of adaptive policy. Useful for known-hot mainnet contracts and diagnostic profiling. - DISABLE_ADAPTIVE_PIN — disables the controller entirely. The branch-cache layer stays active; only the auto-promotion logic is skipped. Bench-validated: with PIN_CONTRACT_TRUNKS in use, controller fires but no promotions occur (the bloat contract's reads go to the pre-pinned tier, not the miss callback). Baseline reproduces byte-for-byte (took=1.22 s, pin_count=89365, divergences=0). --- db/state/execctx/domain_shared.go | 72 ++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 24c4bd5030b..d7bf9755846 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -168,6 +168,16 @@ type SharedDomains struct { // May be nil for test setups whose AggTx doesn't implement // commitment.BranchCacheProvider. branchCache *commitment.BranchCache + + // adaptivePinController owns the policy that decides which + // contracts get pinned based on observed miss pressure. nil when + // branchCache is nil or when the adaptive layer is disabled. The + // controller's miss callback is wired onto branchCache via Bind + // during EnableParaTrieDB; OnBlockComplete fires from sd.Flush. + // Uses sd.txNum as the "block tick" for cooldown semantics — Flush + // is per-batch so the controller's tick maps to batches not blocks, + // but cooldown thresholds are tunable to compensate. + adaptivePinController *commitment.AdaptivePinController } func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger) (*SharedDomains, error) { @@ -209,6 +219,19 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) + // Construct the adaptive pin controller alongside the cache. The + // controller observes per-contract miss pressure and decides + // promotions/extensions/demotions; without binding (deferred to + // EnableParaTrieDB so the db is available for the reader path) + // the controller is inert. + if branchCache != nil && !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { + sd.adaptivePinController = commitment.NewAdaptivePinController( + branchCache, + commitment.DefaultAdaptivePinControllerConfig(), + logger, + ) + } + _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -742,13 +765,34 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { // the local sd.mem value (during the lock) or the cached/MDBX value // (after release). No window where cache is stale vs MDBX. if sd.branchCache != nil { - return sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { + if err := sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { if len(v) == 0 { sd.branchCache.Invalidate(k) return } sd.branchCache.Put(k, v, uint64(step), "sd.Flush") - }) + }); err != nil { + return err + } + // Adaptive controller hook: now that the cache reflects this + // batch's writes, decide promotions / extensions / demotions + // for the contracts whose miss pressure crossed thresholds. + // The reader uses the in-flight Flush tx so pre-cache reads + // see the just-committed bytes (the tx's own writes are + // visible to itself). + if sd.adaptivePinController != nil { + if ttx, ok := tx.(kv.TemporalTx); ok { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil + } + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + } + } + return nil } return sd.mem.Flush(ctx, tx) } @@ -1165,7 +1209,8 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) - // Trigger trunk-preload here (not from NewSharedDomains) so: + // Stage-init is the right firing point for both the env-driven + // forced preload AND the adaptive controller's bind: // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine // with — avoids the cgo-pointer panic that the earlier shared-tx // attempt produced. @@ -1177,11 +1222,24 @@ func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { return } - pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", "") - if pinList == "" { - return + + // Operator-override path: PIN_CONTRACT_TRUNKS forces specific + // contracts to be pinned regardless of adaptive policy. Runs + // first so its pin set is in place before the controller binds. + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + } + + // Bind the adaptive controller to the cache so it starts + // observing miss pressure for un-forced contracts. The reader + // for promote/extend preloads is constructed per-call from the + // in-flight Flush tx (see SD.Flush). + if sd.adaptivePinController != nil { + sd.adaptivePinController.Bind() + sd.logger.Info("[adaptive-pin] enabled", + "max_promoted", commitment.DefaultAdaptivePinControllerConfig().MaxPromotedContracts, + "per_contract_max_mb", commitment.DefaultAdaptivePinControllerConfig().PerContractMaxBudgetBytes/(1<<20)) } - triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) } // SetDeferCommitmentUpdates enables or disables deferred commitment updates. From 2c6ee4ed9582269d4370f710103706ff1306ff7d Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:18:24 +0000 Subject: [PATCH 057/120] commitment: Prometheus metrics for trunk-pin layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the existing internal counters via the diagnostics/metrics package so dashboards can reason about pin effectiveness without log scraping. Six metrics: commitment_branchcache_pinned_hits_total (counter) commitment_branchcache_pinned_misses_total (counter) commitment_branchcache_pinned_entries (gauge) commitment_adaptive_pin_promoted_total (counter) commitment_adaptive_pin_extended_total (counter) commitment_adaptive_pin_demoted_total (counter) commitment_adaptive_pin_active_contracts (gauge) Per-contract labels intentionally omitted — cardinality risk is real (mainnet could see hundreds of promoted contracts over a process lifetime) and the structured [adaptive-pin] log line gives per-contract detail when needed for debugging. Total counters are sufficient for "is this lever working" dashboard signals. Refresh cadence: - pinned-entries gauge updates at SD.Flush via PublishMetrics — once per batch, avoids hot-path cost - AdaptivePinController updates promote/extend/demote/active in OnBlockComplete (also per-batch, same call site) --- db/state/execctx/domain_shared.go | 3 ++ execution/commitment/adaptive_pin.go | 11 +++++ execution/commitment/trunk_pin_metrics.go | 57 +++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 execution/commitment/trunk_pin_metrics.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index d7bf9755846..a55c270c267 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -792,6 +792,9 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) } } + // Refresh pinned-tier gauges once per Flush — once-per-batch + // cadence avoids per-lookup cost in the hot read path. + sd.branchCache.PublishMetrics() return nil } return sd.mem.Flush(ctx, tx) diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 9a21ce1fdde..1cd6b6f6066 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -230,6 +230,17 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } + if promoted > 0 { + mxAdaptivePromoted.AddUint64(uint64(promoted)) + } + if extended > 0 { + mxAdaptiveExtended.AddUint64(uint64(extended)) + } + if demoted > 0 { + mxAdaptiveDemoted.AddUint64(uint64(demoted)) + } + mxAdaptiveActive.SetUint64(uint64(len(c.states))) + if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { c.logger.Info("[adaptive-pin]", "block", blockNum, diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go new file mode 100644 index 00000000000..5bf8e3f8a63 --- /dev/null +++ b/execution/commitment/trunk_pin_metrics.go @@ -0,0 +1,57 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "github.com/erigontech/erigon/diagnostics/metrics" +) + +// Prometheus metrics for the storage-trunk pin layer. Per-contract +// labels intentionally omitted — cardinality risk is real (mainnet +// could see hundreds of promoted contracts over a process lifetime) +// and the structured [adaptive-pin] log line gives per-contract +// detail when needed for debugging. + +var ( + // BranchCache pinned-tier counters. Surface the existing + // PinnedStats hits/misses/entries so dashboards can reason about + // pin effectiveness without scraping logs. + mxPinnedHits = metrics.GetOrCreateCounter("commitment_branchcache_pinned_hits_total") + mxPinnedMisses = metrics.GetOrCreateCounter("commitment_branchcache_pinned_misses_total") + mxPinnedEntries = metrics.GetOrCreateGauge("commitment_branchcache_pinned_entries") + + // AdaptivePinController lifecycle counters. + mxAdaptivePromoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_promoted_total") + mxAdaptiveExtended = metrics.GetOrCreateCounter("commitment_adaptive_pin_extended_total") + mxAdaptiveDemoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_demoted_total") + mxAdaptiveActive = metrics.GetOrCreateGauge("commitment_adaptive_pin_active_contracts") + + // Preload-side counters for diagnostics. Per-contract preload + // duration is too high-cardinality for a histogram label; total + // duration spent in preload is the aggregate proxy. + mxPreloadDurationSecondsTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_duration_seconds_total") + mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") +) + +// publishPinnedGauge updates the pinned-entries gauge from the cache's +// current size. Called from PublishMetrics; cheap so safe to call +// frequently. +func (c *BranchCache) publishPinnedGauge() { + mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) +} + +// PublishMetrics snapshots the cache's current counters into the +// Prometheus registry. Call periodically from the host (e.g. once per +// SD.Flush) to keep metrics fresh — atomic counter loads are cheap +// but not free, so a one-call-per-batch cadence avoids hot-path cost. +func (c *BranchCache) PublishMetrics() { + mxPinnedHits.SetUint64(c.pinnedHits.Load()) + mxPinnedMisses.SetUint64(c.pinnedMisses.Load()) + c.publishPinnedGauge() +} From f48c411a9caf63cc3ea2df03b083a5dc0c67cadf Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:21:15 +0000 Subject: [PATCH 058/120] commitment: tests for trunk-pin layer + delete orphaned warmup test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New trunk_pin_test.go covers the additions: TestPinEntry_AndPinnedCount — basic pin-tier mechanics TestInvalidate_RemovesFromPinned — pin-aware Invalidate TestGetWithOrigin_ChecksPinnedTier — pin-aware GetWithOrigin TestClear_ResetsPinnedTier — pin-aware Clear TestMissCallback_FiresOnTripleMiss — cache hook semantics TestContractHashFromPrefix_* — storage-trunk decode TestContractTrunkPreload_PhasedRun — Initial+Extend split TestAdaptivePinController_PromoteOnThreshold TestAdaptivePinController_DemoteOnCold Updated branch_cache_test.go's TestBranchCache_Stats to match the new Stats string format (added pinned-tier columns; "tail entries" moved past the pin block). Deleted execution/commitment/warmup_cache_test.go — orphaned by the WarmupCache type deletion in commit ded378f789. Test referenced NewWarmupCache which no longer exists; removing unblocks `go test ./execution/commitment/...`. --- execution/commitment/branch_cache_test.go | 4 +- execution/commitment/trunk_pin_test.go | 231 ++++++++++ execution/commitment/warmup_cache_test.go | 523 ---------------------- 3 files changed, 234 insertions(+), 524 deletions(-) create mode 100644 execution/commitment/trunk_pin_test.go delete mode 100644 execution/commitment/warmup_cache_test.go diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index f545a23a650..29e3534e817 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -176,7 +176,9 @@ func TestBranchCache_Stats(t *testing.T) { for _, want := range []string{ "root hit=1 miss=0", "tail hit=1 miss=1", - "tail entries=1", // we put 1 deep entry; root not counted in tail + // Pinned tier added; tail still has 1 entry (the deep Put), + // pinned tier remains empty in this test. + "tail hit=1 miss=1 (50.0%) entries=1", } { require.Contains(t, s, want, "Stats output: %s", s) } diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go new file mode 100644 index 00000000000..edf8ebac6b0 --- /dev/null +++ b/execution/commitment/trunk_pin_test.go @@ -0,0 +1,231 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "context" + "encoding/binary" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeReader returns a deterministic branch for any prefix that has a +// 16-bit bitmap encoded after the 2-byte touchMap. The bitmap is +// derived from the path so children are reproducible across runs. +func fakeReader(saturated bool) CommitmentReader { + return func(prefix []byte) ([]byte, uint64, bool, error) { + // Synthesize a branch: 2 B touchMap (zero) || 2 B bitmap || nothing. + // bitmap: full (0xFFFF) when saturated; sparse (single child idx 0) + // otherwise — produces a narrow trunk that exhausts the BFS quickly. + var bitmap uint16 + if saturated { + bitmap = 0xFFFF + } else { + bitmap = 0x0001 + } + buf := make([]byte, 4) + binary.BigEndian.PutUint16(buf[2:4], bitmap) + return buf, 1, true, nil + } +} + +// TestPinEntry_AndPinnedCount confirms basic pin-tier mechanics. +func TestPinEntry_AndPinnedCount(t *testing.T) { + c := NewBranchCache(8) + require.Equal(t, 0, c.PinnedCount()) + c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, "test") + require.Equal(t, 1, c.PinnedCount()) + + // Pinned entry is hit, not the tail. + data, _, ok := c.Get([]byte{0x12, 0x34, 0x56}) + require.True(t, ok) + require.Equal(t, []byte{0xab, 0xcd}, data) + hits, _, _ := c.PinnedStats() + require.Equal(t, uint64(1), hits) +} + +// TestInvalidate_RemovesFromPinned ensures Invalidate (now pin-aware) +// deletes a pinned entry, not just LRU/root. +func TestInvalidate_RemovesFromPinned(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, "test") + require.Equal(t, 1, c.PinnedCount()) + + c.Invalidate([]byte{0x12, 0x34}) + require.Equal(t, 0, c.PinnedCount()) + _, _, ok := c.Get([]byte{0x12, 0x34}) + require.False(t, ok) +} + +// TestGetWithOrigin_ChecksPinnedTier ensures GetWithOrigin reads +// pinned entries (was a correctness gap pre-fix). +func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, "preload") + data, origin, _, _, ok := c.GetWithOrigin([]byte{0x12, 0x34}) + require.True(t, ok) + require.Equal(t, []byte{0xab}, data) + require.Equal(t, "preload", origin) +} + +// TestClear_ResetsPinnedTier ensures Clear empties pinned + resets +// pinned-stats counters (was a correctness gap pre-fix). +func TestClear_ResetsPinnedTier(t *testing.T) { + c := NewBranchCache(8) + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, "test") + _, _, _ = c.Get([]byte{0x12, 0x34}) // bump pinned hit + c.Clear() + require.Equal(t, 0, c.PinnedCount()) + hits, misses, _ := c.PinnedStats() + require.Equal(t, uint64(0), hits) + require.Equal(t, uint64(0), misses) +} + +// TestMissCallback_FiresOnTripleMiss ensures the callback fires only +// when ALL three tiers (root, pinned, LRU) miss — not on cache hits. +func TestMissCallback_FiresOnTripleMiss(t *testing.T) { + c := NewBranchCache(8) + var fired atomic.Uint64 + c.SetMissCallback(func(prefix []byte) { + fired.Add(1) + }) + + // Hit (pinned): callback should NOT fire. + c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, "test") + _, _, _ = c.Get([]byte{0x12, 0x34}) + require.Equal(t, uint64(0), fired.Load(), "pinned hit must not fire callback") + + // Miss: callback fires. + _, _, _ = c.Get([]byte{0x99, 0x88, 0x77}) + require.Equal(t, uint64(1), fired.Load(), "triple-miss must fire callback") + + // Unbind: subsequent miss does not fire. + c.SetMissCallback(nil) + _, _, _ = c.Get([]byte{0xaa, 0xbb}) + require.Equal(t, uint64(1), fired.Load(), "unbound callback must not fire") +} + +// TestContractHashFromPrefix_DecodesStorageTrunk ensures the helper +// extracts the contract hash from a 33+ B prefix and rejects shorter +// account-trie prefixes. +func TestContractHashFromPrefix_DecodesStorageTrunk(t *testing.T) { + // 33-byte prefix: 1 HP flag + 32 contract bytes. + prefix := make([]byte, 33) + prefix[0] = 0x00 // HP flag + for i := 1; i <= 32; i++ { + prefix[i] = byte(i) + } + hash, ok := ContractHashFromPrefix(prefix) + require.True(t, ok) + for i := 0; i < 32; i++ { + require.Equal(t, byte(i+1), hash[i]) + } + + // Short prefix: account-trie, must be rejected. + _, ok = ContractHashFromPrefix([]byte{0x00, 0x12, 0x34}) + require.False(t, ok) +} + +// TestContractTrunkPreload_PhasedRun confirms initial+extension semantics: +// first Run consumes its budget, second Run continues from where it left. +func TestContractTrunkPreload_PhasedRun(t *testing.T) { + c := NewBranchCache(64) + hash := make([]byte, 32) + for i := range hash { + hash[i] = byte(i) + } + p, err := NewContractTrunkPreload(hash) + require.NoError(t, err) + + reader := fakeReader(false /* sparse */) + + // Initial Run with tiny budget — pins the root and stops. + n1, queueEmpty, err := p.Run(1<<10, reader, c, nil) + require.NoError(t, err) + require.Greater(t, n1, 0, "initial Run must pin at least the root") + require.False(t, queueEmpty, "sparse trie shouldn't exhaust on tiny budget") + totalAfterInitial := p.PinnedTotal() + require.Equal(t, n1, totalAfterInitial) + + // Extension Run continues from where it stopped. + n2, _, err := p.Run(1<<10, reader, c, nil) + require.NoError(t, err) + require.GreaterOrEqual(t, n2, 0) + require.Equal(t, totalAfterInitial+n2, p.PinnedTotal()) + + // Pinned prefixes are tracked for demotion. + require.Equal(t, p.PinnedTotal(), len(p.PinnedPrefixes())) +} + +// TestAdaptivePinController_PromoteOnThreshold confirms a contract +// crossing PromoteThresholdMisses gets promoted on the next +// OnBlockComplete. +func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 3 + cfg.InitialViewBudgetBytes = 1 << 14 // 16 KiB; small but enough for sparse trie + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + // Forge a 33-byte prefix that decodes to a known contract hash. + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 1) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Trigger 3 cache-misses for the same contract. + for i := 0; i < 3; i++ { + _, _, _ = c.Get(prefix) + } + + // OnBlockComplete should promote the contract. + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) + require.Greater(t, c.PinnedCount(), 0) +} + +// TestAdaptivePinController_DemoteOnCold confirms a promoted contract +// gets demoted after DemoteCooldownBlocks consecutive blocks with no +// misses. +func TestAdaptivePinController_DemoteOnCold(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.DemoteCooldownBlocks = 2 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 7) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Hot block: triggers promotion. + _, _, _ = c.Get(prefix) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1) + pinnedAfterPromote := c.PinnedCount() + require.Greater(t, pinnedAfterPromote, 0) + + // Cold blocks: no misses for the contract. After cooldown the + // controller demotes and invalidates the pin set. + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") + + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") + require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") +} diff --git a/execution/commitment/warmup_cache_test.go b/execution/commitment/warmup_cache_test.go deleted file mode 100644 index db62ec6ceb2..00000000000 --- a/execution/commitment/warmup_cache_test.go +++ /dev/null @@ -1,523 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "bytes" - "crypto/rand" - "testing" - - "github.com/stretchr/testify/require" -) - -// TestWarmupCache_Basic tests basic put/get operations -func TestWarmupCache_Basic(t *testing.T) { - cache := NewWarmupCache() - - // Test branch operations - branchKey := []byte("test-branch-key-12345678901234567890") - branchData := []byte("branch-data-content") - cache.PutBranch(branchKey, branchData) - - gotData, found := cache.GetBranch(branchKey) - if !found { - t.Fatal("expected to find branch") - } - if !bytes.Equal(gotData, branchData) { - t.Errorf("branch data mismatch: got %x, want %x", gotData, branchData) - } - - // Test account operations - accountKey := []byte("12345678901234567890") // 20 bytes - accountUpdate := &Update{Flags: BalanceUpdate} - cache.PutAccount(accountKey, accountUpdate) - - gotUpdate, found := cache.GetAccount(accountKey) - if !found { - t.Fatal("expected to find account") - } - if gotUpdate.Flags != BalanceUpdate { - t.Errorf("account flags mismatch: got %v, want %v", gotUpdate.Flags, BalanceUpdate) - } - - // Test storage operations - storageKey := make([]byte, 52) - rand.Read(storageKey) - storageUpdate := &Update{Flags: StorageUpdate, StorageLen: 5} - copy(storageUpdate.Storage[:], "hello") - cache.PutStorage(storageKey, storageUpdate) - - gotStorage, found := cache.GetStorage(storageKey) - if !found { - t.Fatal("expected to find storage") - } - if gotStorage.Flags != StorageUpdate { - t.Errorf("storage flags mismatch") - } - if !bytes.Equal(gotStorage.Storage[:gotStorage.StorageLen], []byte("hello")) { - t.Errorf("storage data mismatch") - } -} - -// TestWarmupCache_NotFound tests cache misses -func TestWarmupCache_NotFound(t *testing.T) { - cache := NewWarmupCache() - - _, found := cache.GetBranch([]byte("nonexistent")) - if found { - t.Error("expected not to find nonexistent branch") - } - - _, found = cache.GetAccount([]byte("12345678901234567890")) - if found { - t.Error("expected not to find nonexistent account") - } - - _, found = cache.GetStorage(make([]byte, 52)) - if found { - t.Error("expected not to find nonexistent storage") - } -} - -// TestWarmupCache_Eviction tests key eviction -func TestWarmupCache_Eviction(t *testing.T) { - cache := NewWarmupCache() - - key := []byte("12345678901234567890") - cache.PutAccount(key, &Update{Flags: BalanceUpdate}) - - // Should find before eviction - _, found := cache.GetAccount(key) - if !found { - t.Fatal("expected to find account before eviction") - } - - // Evict the key - cache.EvictAccount(key) - - // Should not find after eviction - _, found = cache.GetAccount(key) - if found { - t.Error("expected not to find account after eviction") - } -} - -// TestWarmupCache_Clear tests clearing the cache -func TestWarmupCache_Clear(t *testing.T) { - cache := NewWarmupCache() - - // Add some data - cache.PutAccount([]byte("12345678901234567890"), &Update{}) - cache.PutStorage(make([]byte, 52), &Update{}) - cache.PutBranch([]byte("branch"), []byte("data")) - - // Clear - cache.Clear() - - // Should not find anything - _, found := cache.GetAccount([]byte("12345678901234567890")) - if found { - t.Error("expected not to find account after clear") - } -} - -// TestWarmupCache_KeyPadding tests that shorter keys work correctly -func TestWarmupCache_KeyPadding(t *testing.T) { - cache := NewWarmupCache() - - // Test with a key shorter than the fixed size - shortKey := []byte("short") - cache.PutBranch(shortKey, []byte("data")) - - gotData, found := cache.GetBranch(shortKey) - if !found { - t.Fatal("expected to find branch with short key") - } - if !bytes.Equal(gotData, []byte("data")) { - t.Errorf("data mismatch") - } - - // Ensure different short keys don't collide - shortKey2 := []byte("other") - cache.PutBranch(shortKey2, []byte("data2")) - - gotData, found = cache.GetBranch(shortKey) - if !found || !bytes.Equal(gotData, []byte("data")) { - t.Error("first key affected by second key") - } - - gotData2, found := cache.GetBranch(shortKey2) - if !found || !bytes.Equal(gotData2, []byte("data2")) { - t.Error("second key not found or wrong data") - } -} - -// generateTestKeys creates random keys of the given size -func generateTestKeys(n int, size int) [][]byte { - keys := make([][]byte, n) - for i := 0; i < n; i++ { - keys[i] = make([]byte, size) - rand.Read(keys[i]) - } - return keys -} - -// BenchmarkWarmupCache_Branch benchmarks branch key operations -func BenchmarkWarmupCache_Branch(b *testing.B) { - cache := NewWarmupCache() - - // Pre-populate with some data - keys := generateTestKeys(10000, 52) - data := make([]byte, 100) - rand.Read(data) - - for _, key := range keys { - cache.PutBranch(key, data) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetBranch(key) - } -} - -// BenchmarkWarmupCache_Branch_Put benchmarks Put operations -func BenchmarkWarmupCache_Branch_Put(b *testing.B) { - cache := NewWarmupCache() - const keyCount = 10000 - keys := generateTestKeys(keyCount, 52) - data := make([]byte, 100) - rand.Read(data) - - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - cache.PutBranch(keys[i%keyCount], data) - } -} - -// BenchmarkWarmupCache_Account benchmarks account key operations (20 bytes) -func BenchmarkWarmupCache_Account(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(10000, 20) - update := &Update{} - - for _, key := range keys { - cache.PutAccount(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetAccount(key) - } -} - -// BenchmarkWarmupCache_Storage benchmarks storage key operations (52 bytes) -func BenchmarkWarmupCache_Storage(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(10000, 52) - update := &Update{} - - for _, key := range keys { - cache.PutStorage(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - key := keys[i%len(keys)] - cache.GetStorage(key) - } -} - -// BenchmarkWarmupCache_Mixed simulates realistic mixed workload -func BenchmarkWarmupCache_Mixed(b *testing.B) { - cache := NewWarmupCache() - accountKeys := generateTestKeys(1000, 20) - storageKeys := generateTestKeys(5000, 52) - branchKeys := generateTestKeys(2000, 32) - update := &Update{} - data := make([]byte, 100) - - // Pre-populate - for _, key := range accountKeys { - cache.PutAccount(key, update) - } - for _, key := range storageKeys { - cache.PutStorage(key, update) - } - for _, key := range branchKeys { - cache.PutBranch(key, data) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - switch i % 3 { - case 0: - cache.GetAccount(accountKeys[i%len(accountKeys)]) - case 1: - cache.GetStorage(storageKeys[i%len(storageKeys)]) - case 2: - cache.GetBranch(branchKeys[i%len(branchKeys)]) - } - } -} - -// BenchmarkComparison_Map_100k benchmarks map with 100k entries -func BenchmarkComparison_Map_100k(b *testing.B) { - cache := NewWarmupCache() - keys := generateTestKeys(100000, 52) - update := &Update{} - - for _, key := range keys { - cache.PutStorage(key, update) - } - - b.ResetTimer() - b.ReportAllocs() - - for i := 0; b.Loop(); i++ { - cache.GetStorage(keys[i%len(keys)]) - } -} - -// TestWarmupCache_Stats verifies that hit/miss/evict counters are updated -// correctly across the Get/Put/Evict paths and that Stats() returns a -// deterministic format. -func TestWarmupCache_Stats(t *testing.T) { - cache := NewWarmupCache() - - // Branch hits + misses + bytes-served - bk := []byte("k-branch") - bd := []byte("branch-data-12345") - cache.PutBranch(bk, bd) - if _, ok := cache.GetBranch(bk); !ok { - t.Fatal("expected branch hit") - } - if _, ok := cache.GetBranch([]byte("missing-branch")); ok { - t.Fatal("expected branch miss") - } - - // Branch eviction - cache.EvictBranch(bk) - if _, ok := cache.GetBranch(bk); ok { - t.Fatal("expected branch evicted miss") - } - - // Account hits + misses - ak := []byte("k-acct") - cache.PutAccount(ak, &Update{}) - if _, ok := cache.GetAccount(ak); !ok { - t.Fatal("expected account hit") - } - if _, ok := cache.GetAccount([]byte("missing-acct")); ok { - t.Fatal("expected account miss") - } - - // Storage hits + misses - sk := []byte("k-stor") - cache.PutStorage(sk, &Update{}) - if _, ok := cache.GetStorage(sk); !ok { - t.Fatal("expected storage hit") - } - if _, ok := cache.GetStorage([]byte("missing-stor")); ok { - t.Fatal("expected storage miss") - } - - // Counters: branch hit=1 miss=1 evict=1 bytes=17, acct hit=1 miss=1, stor hit=1 miss=1 - if got := cache.branchHits.Load(); got != 1 { - t.Fatalf("branchHits: got %d, want 1", got) - } - if got := cache.branchMisses.Load(); got != 1 { - t.Fatalf("branchMisses: got %d, want 1", got) - } - if got := cache.branchEvicted.Load(); got != 1 { - t.Fatalf("branchEvicted: got %d, want 1", got) - } - if got := cache.branchBytesServed.Load(); got != uint64(len(bd)) { - t.Fatalf("branchBytesServed: got %d, want %d", got, len(bd)) - } - if got := cache.accountHits.Load(); got != 1 { - t.Fatalf("accountHits: got %d, want 1", got) - } - if got := cache.accountMisses.Load(); got != 1 { - t.Fatalf("accountMisses: got %d, want 1", got) - } - if got := cache.storageHits.Load(); got != 1 { - t.Fatalf("storageHits: got %d, want 1", got) - } - if got := cache.storageMisses.Load(); got != 1 { - t.Fatalf("storageMisses: got %d, want 1", got) - } - - // Stats() format - stats := cache.Stats() - for _, want := range []string{ - "branch hit=1 miss=1 evict=1", - "acct hit=1 miss=1", - "stor hit=1 miss=1", - } { - if !bytes.Contains([]byte(stats), []byte(want)) { - t.Errorf("Stats() missing %q\nfull: %s", want, stats) - } - } - - // ResetStats zeros counters but leaves data intact - cache.ResetStats() - if got := cache.branchHits.Load(); got != 0 { - t.Fatalf("branchHits after Reset: got %d, want 0", got) - } - if _, ok := cache.GetAccount(ak); !ok { - t.Fatal("account data should survive ResetStats") - } -} - -// TestWarmupCache_DirtyFlag verifies the dirty flag scaffolding — -// MarkBranchDirty + PutBranchIfClean — that prevents late writers from -// clobbering fresh data once cross-block persistence is wired up. -func TestWarmupCache_DirtyFlag(t *testing.T) { - cache := NewWarmupCache() - key := []byte("k-branch") - v1 := []byte("first-value") - v2 := []byte("second-value-skipped") - - // First Put always succeeds (no existing entry to be dirty). - require.True(t, cache.PutBranchIfClean(key, v1)) - got, ok := cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v1, got) - - // Marking dirty doesn't remove the entry — Get still returns it. - cache.MarkBranchDirty(key) - got, ok = cache.GetBranch(key) - require.True(t, ok, "MarkBranchDirty should not evict; consumers decide what to do with dirty data") - require.Equal(t, v1, got) - - // PutBranchIfClean refuses to overwrite a dirty entry. - require.False(t, cache.PutBranchIfClean(key, v2), - "PutBranchIfClean must skip when entry is dirty") - got, ok = cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v1, got, "dirty value must be preserved (write was rejected)") - - // PutBranch (non-If-Clean variant) overwrites unconditionally — used - // by the fold encoder path that holds the canonical new value. - cache.PutBranch(key, v2) - got, ok = cache.GetBranch(key) - require.True(t, ok) - require.Equal(t, v2, got) - - // After unconditional overwrite, the new entry is no longer dirty - // (fresh entry replaces the dirty one). - require.True(t, cache.PutBranchIfClean(key, []byte("third-value")), - "PutBranchIfClean must accept after unconditional Put cleared the dirty entry") -} - -// TestWarmupCache_DirtyFlag_MarkAbsentKey verifies that marking a key -// dirty when no entry exists is a no-op (no panic, no entry created). -func TestWarmupCache_DirtyFlag_MarkAbsentKey(t *testing.T) { - cache := NewWarmupCache() - cache.MarkBranchDirty([]byte("never-stored")) - _, ok := cache.GetBranch([]byte("never-stored")) - require.False(t, ok, "MarkBranchDirty on absent key should not create an entry") - - // PutBranchIfClean for the same absent key should succeed (no dirty - // entry exists, so the write proceeds). - require.True(t, cache.PutBranchIfClean([]byte("never-stored"), []byte("v"))) -} - -// TestWarmupCache_GetBranchDecoded verifies the lazy-decode read path: -// stored encoded bytes are decoded on first decoded-read and cached for -// subsequent reads, returning cells equivalent to direct DecodeBranchInto. -func TestWarmupCache_GetBranchDecoded(t *testing.T) { - cache := NewWarmupCache() - - // Encode a branch the same way BranchEncoder would, so our test - // data has the canonical [touchMap | bitmap | cells...] layout. - row, bm := generateCellRow(t, 16) - be := NewBranchEncoder(1024) - cellData := generateCellEncodeDataRow(t, row, bm) - enc, err := be.EncodeBranch(bm, bm, bm, &cellData) - require.NoError(t, err) - - prefix := []byte{0x12, 0x34} - cache.PutBranch(prefix, enc) - - // First decoded-read decodes lazily. - bitmap, cells, ok := cache.GetBranchDecoded(prefix) - require.True(t, ok) - require.Equal(t, bm, bitmap) - require.NotNil(t, cells) - - // Cells should match what direct DecodeBranchInto would produce - // from the same encoded bytes (skip the 2-byte touchMap prefix). - var expected [16]cell - expectedMaps, err := DecodeBranchInto(enc[2:], false, &expected) - require.NoError(t, err) - require.Equal(t, expectedMaps.Bitmap, bitmap) - for i := range cells { - require.Equal(t, expected[i].extLen, cells[i].extLen, "cell %d extLen", i) - require.Equal(t, expected[i].extension[:expected[i].extLen], cells[i].extension[:cells[i].extLen], "cell %d extension", i) - require.Equal(t, expected[i].accountAddr[:expected[i].accountAddrLen], cells[i].accountAddr[:cells[i].accountAddrLen], "cell %d accountAddr", i) - require.Equal(t, expected[i].storageAddr[:expected[i].storageAddrLen], cells[i].storageAddr[:cells[i].storageAddrLen], "cell %d storageAddr", i) - require.Equal(t, expected[i].hash[:expected[i].hashLen], cells[i].hash[:cells[i].hashLen], "cell %d hash", i) - } - - // Second decoded-read returns the SAME cells pointer — lazy decode - // runs at most once per entry. - bitmap2, cells2, ok := cache.GetBranchDecoded(prefix) - require.True(t, ok) - require.Equal(t, bm, bitmap2) - require.Same(t, cells, cells2, "expected cached cells pointer to be reused") - - // Encoded form is unchanged after decoded reads. - encGot, ok := cache.GetBranch(prefix) - require.True(t, ok) - require.Equal(t, []byte(enc), encGot, "encoded form unchanged by decoded reads") -} - -// TestWarmupCache_GetBranchDecoded_Miss verifies that misses on -// GetBranchDecoded behave the same as misses on GetBranch. -func TestWarmupCache_GetBranchDecoded_Miss(t *testing.T) { - cache := NewWarmupCache() - _, _, ok := cache.GetBranchDecoded([]byte("never-stored")) - require.False(t, ok) -} - -// TestWarmupCache_GetBranchDecoded_TruncatedData verifies the decode-error -// path — corrupt entry returns ok=false rather than panicking. -func TestWarmupCache_GetBranchDecoded_TruncatedData(t *testing.T) { - cache := NewWarmupCache() - // One byte is shorter than the touchMap prefix; decode will error. - cache.PutBranch([]byte("k"), []byte{0x42}) - _, _, ok := cache.GetBranchDecoded([]byte("k")) - require.False(t, ok, "truncated entry should return ok=false, not panic") - - // Encoded form still retrievable for callers that don't need decode. - got, ok := cache.GetBranch([]byte("k")) - require.True(t, ok) - require.Equal(t, []byte{0x42}, got) -} From e69e9bfbacd84645f5665433c949464c4a4b1f9f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:26:48 +0000 Subject: [PATCH 059/120] commitment, db/state: drop bench-only env knobs and cache-fp log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup pass before PR. Removes scaffolding that supported the SSTORE-bloat investigation but isn't operator-relevant for production: Env knobs removed: - TRUNK_PROBE — logged contract hashes of depth-64 commitment prefixes on first sight. Was used to identify the bloat contract; now obsolete. - DISABLE_BRANCH_CACHE_READS — A/B switch for cache-vs-no-cache benchmarking. The cache is now a fundamental architecture component; DISABLE_ADAPTIVE_PIN already covers the operator-safety use case. - BRANCH_CACHE_FINGERPRINT — gated the [commitment][cache-fp] log line that emitted ~30 fields of investigation-era counters per block. Replaced by Prometheus metrics and the [adaptive-pin] log. - Dead variables in hex_patricia_hashed.go: verifyBranchCache and disableBranchCacheReads were declared but never referenced (callers were removed in the WarmupCache deletion). Code removed: - The 90-line [cache-fp] log block in commitmentdb/commitment_context.go that referenced ~10 bench-only counter helpers (SkipLoadResetCounters, ContainsHashStats, SstoreClassificationCounts, WarmerBranchOutcomeStats, FileReadCount, UniqueLenBuckets, etc.) - The trunk-probe path in db/state/changeset/state_changeset.go - Stale comment references to verifyBranchCache The bench-only counter helpers themselves (SkipLoadResetCounters etc.) are now unreferenced but left in place — a follow-up cleanup commit can remove them once we confirm no other callers exist. --- db/state/changeset/state_changeset.go | 16 --- db/state/execctx/domain_shared.go | 10 +- execution/commitment/branch_cache.go | 6 ++ .../commitmentdb/commitment_context.go | 102 ------------------ execution/commitment/hex_patricia_hashed.go | 19 ---- execution/commitment/trunk_pin_metrics.go | 26 ++--- 6 files changed, 21 insertions(+), 158 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 04b0667a95d..0e30f687897 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -18,7 +18,6 @@ package changeset import ( "encoding/binary" - "encoding/hex" "fmt" "math" "strings" @@ -28,19 +27,11 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/length" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/execution/types/accounts" ) -// logTrunkProbe is set by TRUNK_PROBE=true to log the keccak256 hash of -// each contract whose storage subtree root (depth-64 commitment prefix, -// 33 bytes) is read from the file layer for the first time. The bytes -// after the 0x00 flag byte ARE the contract's keccak256(addr), which -// is what PIN_CONTRACT_TRUNKS expects in its hex form. -var logTrunkProbe = dbg.EnvBool("TRUNK_PROBE", false) - type StateChangeSet struct { Diffs [kv.DomainLen]kv.DomainDiff // there are 4 domains of state changes } @@ -639,13 +630,6 @@ func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, sta _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) bucket := lenBucket(len(key)) - // Trunk-probe log: depth-64 commitment prefix on first sight. - // Format key as 0x00 || keccak256(addr); the trailing 32 bytes are - // the value to feed to PIN_CONTRACT_TRUNKS for that contract. - if logTrunkProbe && !alreadySeen && bucket == 5 && domain == kv.CommitmentDomain && len(key) == 33 && key[0] == 0x00 { - log.Info("[trunk-probe] new depth-64 commitment prefix", "contract_hash", hex.EncodeToString(key[1:33])) - } - dm.Lock() defer dm.Unlock() dm.FileReadCount++ diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a55c270c267..da2b9a97610 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -210,12 +210,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg if p, ok := tx.AggTx().(commitment.BranchCacheProvider); ok { branchCache = p.BranchCache() } - // DISABLE_BRANCH_CACHE_READS gates the cache out for A/B benchmarking. - // When set, sd.branchCache stays nil so sd.GetLatest skips the cache - // layer entirely and every CommitmentDomain read goes to MDBX. - if dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) { - branchCache = nil - } sd.branchCache = branchCache sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tv, tx.Debug().Dirs().Tmp, branchCache) @@ -647,8 +641,8 @@ func (sd *SharedDomains) GetStateCache() *cache.StateCache { // the parent SD's mem, and direct MDBX via tx.GetLatest) and returns // the bytes from each. Read-only, intended for divergence-detection // diagnostics — pinpoints which layer holds bytes that disagree with -// the BranchCache when verifyBranchCache is on. Bytes are copied so -// the caller can hold them past tx lifetime. +// the BranchCache. Bytes are copied so the caller can hold them past +// tx lifetime. func (sd *SharedDomains) ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { if v, _, ok := sd.mem.GetLatest(domain, key); ok { memOk = true diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index d7c22d2e20d..c07f25ea189 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -200,6 +200,12 @@ type BranchCache struct { // when no callback is installed. onMiss atomic.Pointer[MissCallback] + // Last-published Prometheus counter snapshots. PublishMetrics + // emits the delta between current and last so the monotonic + // counters track real activity per Flush, not snapshot absolutes. + lastPublishedPinnedHits atomic.Uint64 + lastPublishedPinnedMisses atomic.Uint64 + // preloadClaimed is set the first time TryClaimPreload is called. // Used by the trunk-preload trigger (PIN_CONTRACT_TRUNKS hook in // SharedDomains construction) so the preload goroutine fires diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 5a97bceed4b..7a95675b315 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,7 +18,6 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/datastruct/existence" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" @@ -33,13 +32,6 @@ import ( var ( mxCommitmentRunning = metrics.GetOrCreateGauge("domain_running_commitment") mxCommitmentTook = metrics.GetOrCreateSummary("domain_commitment_took") - - // logCacheFingerprint, when true, makes ComputeCommitment emit a - // "[cache-fp]" log line at end-of-block with (block, root, fp, - // divergences). Diff two builds' logs offline to localise the first - // block at which their BranchCache states diverge. Off by default — - // gate via env BRANCH_CACHE_FINGERPRINT=true. - logCacheFingerprint = dbg.EnvBool("BRANCH_CACHE_FINGERPRINT", false) ) type sd interface { @@ -335,100 +327,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) - // Per-block BranchCache fingerprint — gated by BRANCH_CACHE_FINGERPRINT - // env. Emit a single log line of (block, root, cache_fp, divergences) - // so two builds running the same workload can be diffed offline to - // localise the first block at which their caches diverge. See - // commitment.BranchCache.Fingerprint for the hash semantics. - // took + keys exposed at Info so the calculator's per-block compute - // time is visible without enabling debug logs (used by the - // snapshot-vs-mdbx perf-equivalence investigation to attribute - // the gap inside newPayload). - if logCacheFingerprint { - if hph, ok := sdc.patriciaTrie.(*commitment.HexPatriciaHashed); ok { - if bc := hph.BranchCache(); bc != nil { - // Skip/load/reset counters are process-cumulative; the - // per-block delta requires subtracting consecutive lines. - // Used to answer: "what fraction of computeCellHash calls - // actually fetched the underlying value vs reused a - // memoized stateHash?" - load, skipped, reset, diskSto, diskAcc := commitment.SkipLoadResetCounters() - hasStoMiss := commitment.HasStorageMissCount() - xfTrue, xfFalse := existence.ContainsHashStats() - ssIns, ssUpd, ssDel, _ := commitment.SstoreClassificationCounts() - wHit, wEmpty := commitment.WarmerBranchOutcomeStats() - // Per-domain file-read counts: lets us decompose the - // aggregate `files=N` from [domain reads] into Commitment - // (branch reads), Storage (value loads), Account, Code. - // All cumulative; deltas via successive lines. - var aFiles, sFiles, cFiles, mFiles int64 - var aUniq, sUniq, cUniq, mUniq int64 - var mLens [10]int64 - if m := sdc.sharedDomains.Metrics(); m != nil { - m.RLock() - if d := m.Domains[kv.AccountsDomain]; d != nil { - aFiles = d.FileReadCount - aUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.StorageDomain]; d != nil { - sFiles = d.FileReadCount - sUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.CodeDomain]; d != nil { - cFiles = d.FileReadCount - cUniq = d.UniqueFileReadCount - } - if d := m.Domains[kv.CommitmentDomain]; d != nil { - mFiles = d.FileReadCount - mUniq = d.UniqueFileReadCount - mLens = d.UniqueLenBuckets - } - m.RUnlock() - } - // commLens shows depth distribution. Buckets within the storage - // subtree (33B+) are sub-divided to distinguish per-contract - // storage trunk (33B / 34-36B) from leaf-parent depths - // (45-64B). See state_changeset.go UniqueLenBuckets. - commLens := fmt.Sprintf( - "1B:%d|2-4B:%d|5-8B:%d|9-16B:%d|17-32B:%d|33B:%d|34-36B:%d|37-44B:%d|45-64B:%d|>64B:%d", - mLens[0], mLens[1], mLens[2], mLens[3], mLens[4], - mLens[5], mLens[6], mLens[7], mLens[8], mLens[9]) - pinHits, pinMisses, pinEntries := bc.PinnedStats() - log.Info("[commitment][cache-fp]", - "block", blockNum, - "root", hex.EncodeToString(rootHash), - "fp", fmt.Sprintf("%016x", bc.Fingerprint()), - "divergences", bc.VerifyDivergences(), - "took", took, - "keys", common.PrettyCounter(updateCount), - "load", load, - "skipped", skipped, - "reset", reset, - "disk_sto", diskSto, - "disk_acc", diskAcc, - "has_sto_miss", hasStoMiss, - "xf_true", xfTrue, - "xf_false", xfFalse, - "ss_ins", ssIns, - "ss_upd", ssUpd, - "ss_del", ssDel, - "w_hit", wHit, - "w_empty", wEmpty, - "files_acc", aFiles, - "files_sto", sFiles, - "files_code", cFiles, - "files_comm", mFiles, - "uniq_acc", aUniq, - "uniq_sto", sUniq, - "uniq_code", cUniq, - "uniq_comm", mUniq, - "comm_lens", commLens, - "pin_hit", pinHits, - "pin_miss", pinMisses, - "pin_count", pinEntries) - } - } - } }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index ce4bb76cd84..eedacefab08 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2994,25 +2994,6 @@ func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } -// verifyBranchCache, when true, makes branchFromCacheOrDB cross-check -// every BranchCache hit against ctx.Branch and record a divergence on -// the cache when the bytes disagree. Off by default — gate via env -// BRANCH_CACHE_VERIFY=true. Use during cross-block-cache-lifetime -// investigations to localise a bad cached entry to the read site -// instead of waiting for the downstream wrong-trie-root to surface -// many blocks later. -var verifyBranchCache = dbg.EnvBool("BRANCH_CACHE_VERIFY", false) - -// disableBranchCacheReads forces branchFromCacheOrDB to skip the L2 -// BranchCache read path — every read goes to ctx.Branch (sd.mem → -// MDBX). Cache writes (CollectUpdate.Put, branchFromCacheOrDB.Put on -// L3 fallback) still fire so verify-mode can keep comparing cache vs -// canonical. Use to A/B test correctness with-cache vs without-cache -// without recompiling. Confirmed in 2026-05-06 investigation that -// flipping this distinguishes "cache holds bad data" (bench passes -// further) from "deeper compute bug" (bench fails at the same block). -var disableBranchCacheReads = dbg.EnvBool("DISABLE_BRANCH_CACHE_READS", false) - // branchFromCacheOrDB reads branch data via ctx.Branch, which goes // through sd.mem -> sd.parent.mem -> aggregator-scope BranchCache -> // MDBX. The HPH-side warmup cache layer that used to sit above this diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go index 5bf8e3f8a63..82c6b2e26fc 100644 --- a/execution/commitment/trunk_pin_metrics.go +++ b/execution/commitment/trunk_pin_metrics.go @@ -39,19 +39,19 @@ var ( mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") ) -// publishPinnedGauge updates the pinned-entries gauge from the cache's -// current size. Called from PublishMetrics; cheap so safe to call -// frequently. -func (c *BranchCache) publishPinnedGauge() { - mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) -} - // PublishMetrics snapshots the cache's current counters into the -// Prometheus registry. Call periodically from the host (e.g. once per -// SD.Flush) to keep metrics fresh — atomic counter loads are cheap -// but not free, so a one-call-per-batch cadence avoids hot-path cost. +// Prometheus registry. Counters are monotonic-add so the cache tracks +// last-published values internally and emits deltas. Gauges Set +// absolute. Call periodically from the host (e.g. once per SD.Flush); +// the once-per-batch cadence avoids hot-path cost. func (c *BranchCache) PublishMetrics() { - mxPinnedHits.SetUint64(c.pinnedHits.Load()) - mxPinnedMisses.SetUint64(c.pinnedMisses.Load()) - c.publishPinnedGauge() + hits := c.pinnedHits.Load() + misses := c.pinnedMisses.Load() + if delta := hits - c.lastPublishedPinnedHits.Swap(hits); delta > 0 { + mxPinnedHits.AddUint64(delta) + } + if delta := misses - c.lastPublishedPinnedMisses.Swap(misses); delta > 0 { + mxPinnedMisses.AddUint64(delta) + } + mxPinnedEntries.SetUint64(uint64(c.pinned.Len())) } From e0b01892a7224281373e971cb27481a90b4e7ef2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 10 May 2026 14:31:54 +0000 Subject: [PATCH 060/120] lint: gofmt fixes + drop snapshot_vs_mdbx bench test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply gofmt to four files where the perf-stack work left formatting differences from main's conventions. None are functionally meaningful; all are whitespace. Drop db/state/snapshot_vs_mdbx_bench_test.go — 714-line bench test file from the snapshot-vs-MDBX equivalence investigation (snapshot-vs-mdbx-performance-equivalence.md memo). Out of scope for this PR; the bench harness can return as a standalone artefact when that investigation resumes. --- common/dbg/experiments.go | 2 +- db/datastruct/existence/existence_filter.go | 1 + db/state/snapshot_vs_mdbx_bench_test.go | 714 ------------------ .../hex_concurrent_patricia_hashed.go | 1 + execution/commitment/hex_patricia_hashed.go | 5 +- 5 files changed, 6 insertions(+), 717 deletions(-) delete mode 100644 db/state/snapshot_vs_mdbx_bench_test.go diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index b72c956181d..0d34c5a76b7 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -119,7 +119,7 @@ var ( BorValidateHeaderTime = EnvBool("BOR_VALIDATE_HEADER_TIME", true) TraceDeletion = EnvBool("TRACE_DELETION", false) - RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) + RpcDropResponse = EnvBool("RPC_DROP_RESPONSE", false) // The original default was NumCPU()*8 with the rationale "io-bound, more // workers drive more concurrent I/O." Our measurements (cold-cgroup 6c // vs unconstrained 12c on the SSTORE-bloat bench) show read_bytes diff --git a/db/datastruct/existence/existence_filter.go b/db/datastruct/existence/existence_filter.go index 78e85db1a70..32230e73d6c 100644 --- a/db/datastruct/existence/existence_filter.go +++ b/db/datastruct/existence/existence_filter.go @@ -80,6 +80,7 @@ func (b *Filter) AddHash(hash uint64) error { b.filter.AddHash(hash) return nil } + // Process-cumulative counters of ContainsHash outcomes. // True = "probably present" (filter does not short-circuit, lookup proceeds). // False = "definitely not present" (filter short-circuits the lookup). diff --git a/db/state/snapshot_vs_mdbx_bench_test.go b/db/state/snapshot_vs_mdbx_bench_test.go deleted file mode 100644 index c2efacdcab9..00000000000 --- a/db/state/snapshot_vs_mdbx_bench_test.go +++ /dev/null @@ -1,714 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package state_test - -// Benchmarks for the "Snapshot vs MDBX read-cost equivalence" -// investigation. See memory: snapshot-vs-mdbx-performance-equivalence.md -// -// Premise: with the OS page cache warm, a key lookup served from -// MDBX vs from a snapshot .kv file should cost roughly the same — the -// disk and page cache are identical. Today they don't. These benches -// quantify the gap (H0) and decompose it (H1-H4). -// -// **Status: SCAFFOLD.** Compiles and runs (skipped) so the structure -// survives. Real-datadir bootstrap and synthetic-fixture bootstrap are -// the two TODOs that turn this into a measurement tool. -// -// Two operating modes are anticipated: -// -// 1. **Synthetic mode** (preferred for CI / reproducibility): -// use testDbAndAggregatorBench, write K1 keys at low txNums, -// buildFiles+prune so they land in files, write K2 at high -// txNums and leave in MDBX. No external datadir required. -// -// 2. **Real-datadir mode** (preferred for fidelity): -// open an existing perf-devnet-3 / mainnet datadir read-only, -// pick keys by cursor-iterating the values table (mdbxKeys) and -// walking the latest .kv decompressor (fileKeys). Matches -// production file-count, Bloom false-positive rate, etc. -// -// Both fail with b.Skip() in this scaffold; the TODOs below are the -// agenda for turning it into a real measurement. - -import ( - "encoding/binary" - "flag" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/erigontech/erigon/common/length" - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/db/datadir" - "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/kv/dbcfg" - "github.com/erigontech/erigon/db/kv/mdbx" - "github.com/erigontech/erigon/db/kv/temporal" - "github.com/erigontech/erigon/db/state" - "github.com/erigontech/erigon/db/state/execctx" -) - -var snapDataDir = flag.String("snapdatadir", "", - "path to a real datadir (chaindata + snapshots) for snapshot-vs-mdbx benches; "+ - "when empty, real-datadir benches are skipped and synthetic mode runs") - -// snapBenchSetup carries everything a sub-bench needs: an open RO tx, -// the AggregatorRoTx for Debug* paths, the domain under test, and the -// pre-picked key sets. Set up once per Benchmark function so warmup -// and timing happen on the same data. -type snapBenchSetup struct { - tx kv.TemporalTx - aggTx *state.AggregatorRoTx - domain kv.Domain - mdbxKeys [][]byte - fileKeys [][]byte -} - -// openSnapBench builds a snapBenchSetup for the given domain. Picks -// real-datadir mode iff --snapdatadir is set, otherwise synthetic. -// Both paths currently call b.Skip; the TODOs below are the work. -func openSnapBench(b *testing.B, domain kv.Domain) *snapBenchSetup { - b.Helper() - if *snapDataDir != "" { - return openSnapBenchRealDatadir(b, domain, *snapDataDir) - } - return openSnapBenchSynthetic(b, domain) -} - -// openSnapBenchRealDatadir opens an existing chaindata+snapshots -// datadir read-only and picks keys from production files. -// -// Uses MDBX Accede mode (don't create, must already exist) and -// state.New(...).MustOpen which loads the on-disk schema. ResolveDB -// settings reads stepSize / stepsInFrozenFile from the existing DB. -// -// CAUTION: opening a chaindata that another erigon process is writing -// to is risky. Bench against an idle copy. The script that drives -// these benches typically does prepare-run.sh first to snapshot a -// fresh chaindata copy under /erigon-data/perf-devnet-3-run/ or -// similar. Pass that path via --snapdatadir. -func openSnapBenchRealDatadir(b *testing.B, domain kv.Domain, datadirPath string) *snapBenchSetup { - b.Helper() - logger := log.New() - dirs := datadir.New(datadirPath) - - rawDB := mdbx.New(dbcfg.ChainDB, logger). - Path(dirs.Chaindata). - Accede(true). // must exist; do not create - MustOpen() - b.Cleanup(rawDB.Close) - - settings, err := state.ResolveErigonDBSettings(dirs, logger, false /*noDownloader*/) - require.NoError(b, err, "ResolveErigonDBSettings") - - agg := state.New(dirs). - Logger(logger). - WithErigonDBSettings(settings). - MustOpen(b.Context(), rawDB) - require.NoError(b, agg.OpenFolder(), "agg.OpenFolder") - b.Cleanup(agg.Close) - - tdb, err := temporal.New(rawDB, agg) - require.NoError(b, err) - b.Cleanup(tdb.Close) - - tx, err := tdb.BeginTemporalRo(b.Context()) - require.NoError(b, err) - b.Cleanup(func() { tx.Rollback() }) - - aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) - require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") - - mdbxKeys, fileKeys := pickRealDatadirKeys(b, aggTx, tx, domain, 10_000) - require.NotEmpty(b, fileKeys, "no file-resident keys found in %s for domain %v", datadirPath, domain) - if len(mdbxKeys) == 0 { - // Heavily pruned production datadir — every MDBX row is - // step-shadowed by a file. The MDBX-side sub-benches will skip - // (runH0 short-circuits on empty key set). Compare File_path - // against the synthetic MDBX_path baseline; clearly note the - // cross-mode caveat in the issue write-up. - b.Logf("real-datadir: no MDBX-resident keys (datadir is fully pruned). " + - "File_path numbers are real; cross-reference MDBX_path against the synthetic bench.") - } - b.Logf("real-datadir keys: mdbx=%d file=%d (domain=%v, path=%s)", - len(mdbxKeys), len(fileKeys), domain, datadirPath) - - return &snapBenchSetup{ - tx: tx, - aggTx: aggTx, - domain: domain, - mdbxKeys: mdbxKeys, - fileKeys: fileKeys, - } -} - -// pickRealDatadirKeys walks the existing datadir to find up to -// `target` MDBX-resident and `target` file-resident keys for the -// given domain. -// -// Strategy A — cursor-walk the per-domain values table for -// MDBX-resident keys (those rows ARE in MDBX by definition). Strip -// the trailing 8-byte inverted-step from each row's key. -// -// Strategy B — for file-resident keys, randomly sample address-sized -// or storage-key-sized byte strings from the cursor walk's MDBX keys -// (deterministic via seeded RNG): for each candidate, query -// DebugGetLatestFromFiles. If the file path returns ok and MDBX does -// NOT, the key is file-resident. This is approximate — keys that -// exist BOTH in MDBX and files (during prune transitions) are -// classified as MDBX-resident, which is what we want for bench -// timing accuracy (MDBX always wins on getLatest). -// -// Practical note: in production datadirs the MDBX values table -// usually has fewer rows than the snapshot files combined, so we -// expect mdbxKeys to fill faster than fileKeys. To find file-only -// keys we'd need to walk the .kv files directly via seg.Decompressor -// — that's a follow-on once Strategy B is shown to be insufficient. -func pickRealDatadirKeys(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, target int) (mdbxKeys, fileKeys [][]byte) { - b.Helper() - table := domainValsTable(b, domain) - - c, err := tx.Cursor(table) - require.NoError(b, err) - defer c.Close() - - const stepSuffixLen = 8 // values-table key = realKey || invertedStep - seen := make(map[string]struct{}) - var rowsScanned, dbOkCount int - - // First pass: collect MDBX-resident keys directly from the values table. - for k, _, err := c.First(); k != nil; k, _, err = c.Next() { - require.NoError(b, err) - rowsScanned++ - if len(k) <= stepSuffixLen { - continue // unexpected; skip - } - realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) - ks := string(realKey) - if _, dup := seen[ks]; dup { - continue - } - seen[ks] = struct{}{} - - // Confirm MDBX residency via DebugGetLatestFromDB so we don't - // include keys that, despite having a row here, don't actually - // resolve in the read path (edge cases around step bounds). - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) - require.NoError(b, err) - if dbOk { - dbOkCount++ - mdbxKeys = append(mdbxKeys, realKey) - if len(mdbxKeys) >= target { - break - } - } - } - b.Logf("first-pass scan: table=%s rows=%d unique_keys=%d dbOk=%d", - table, rowsScanned, len(seen), dbOkCount) - - // Second pass: walk the same cursor again looking for keys that - // MISS MDBX but HIT files. Iterating the values table catches - // keys that left a row but have all data in files (delete - // markers, post-prune residue), which is approximate but cheap. - // For a full file-only walk we'd open the .kv decompressor; this - // is good enough for the H0 first cut. - for k, _, err := c.First(); k != nil; k, _, err = c.Next() { - if len(fileKeys) >= target { - break - } - require.NoError(b, err) - if len(k) <= stepSuffixLen { - continue - } - realKey := append([]byte(nil), k[:len(k)-stepSuffixLen]...) - - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, realKey, tx) - require.NoError(b, err) - if dbOk { - continue // MDBX already serves this; not file-only - } - _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, realKey, 0) - require.NoError(b, err) - if fileOk { - fileKeys = append(fileKeys, realKey) - } - } - - return mdbxKeys, fileKeys -} - -// domainValsTable returns the MDBX values-table name for the given -// domain. Used by pickRealDatadirKeys to cursor-walk MDBX directly. -func domainValsTable(b *testing.B, domain kv.Domain) string { - b.Helper() - switch domain { - case kv.AccountsDomain: - return kv.TblAccountVals - case kv.StorageDomain: - return kv.TblStorageVals - case kv.CodeDomain: - return kv.TblCodeVals - case kv.CommitmentDomain: - return kv.TblCommitmentVals - default: - b.Fatalf("domainValsTable: domain %v not mapped", domain) - return "" - } -} - -// openSnapBenchSynthetic builds an in-memory aggregator with a known -// key distribution and returns a setup whose two key sets are -// guaranteed to live on the file path and the MDBX path respectively. -// -// Sequence: -// -// 1. testDbAndAggregatorBench builds a fresh agg + tempdir. -// 2. Phase 1 — write keysetSize "old" keys at txNums spanning steps -// [0..stepsToFlush-1], deterministically generated from seed -// phase1Seed so the same bench reproduces. -// 3. Flush + commit + agg.BuildFiles(stepsToFlush*aggStep) so those -// keys land in .kv files. PruneSmallBatches removes them from -// MDBX. After this, phase-1 keys are file-resident. -// 4. Phase 2 — write keysetSize "new" keys at txNums spanning steps -// [stepsToFlush..stepsToFlush+1], from disjoint phase2Seed so no -// overlap with phase 1. NO BuildFiles for these. They stay in -// MDBX. -// 5. Begin a RO temporal tx, return setup. -// -// We verify each key actually lives where we expect via -// DebugGetLatestFromDB / DebugGetLatestFromFiles — fail loudly if a -// key isn't where the test plan says it should be (catches setup -// regressions). -func openSnapBenchSynthetic(b *testing.B, domain kv.Domain) *snapBenchSetup { - b.Helper() - - const ( - // aggStep small + many full steps mirrors how the fuzz test sets - // up data. BuildFiles only builds completed steps, so phase-1 has - // to span an integer number of full steps. - aggStep = 16 - stepsToFlush = 64 // phase 1 spans this many full steps - keysetSize = stepsToFlush * aggStep // = 1024; one key per txNum, every step is full - phase1Seed = 0xC0FFEE - phase2Seed = 0xDECAFB - // phase-2 starts well past the built step boundary so its keys - // can't be lumped into a future build. - phase2BaseTxNum = uint64(stepsToFlush) * aggStep - ) - - // keySize varies per domain; values are 8-byte counters (txNum - // encoded big-endian) so put/lookup paths are exercised. - var keySize int - switch domain { - case kv.AccountsDomain: - keySize = length.Addr - case kv.StorageDomain: - keySize = length.Addr + length.Hash - default: - b.Fatalf("openSnapBenchSynthetic: domain %v not yet wired", domain) - } - - db, agg := testDbAndAggregatorBench(b, aggStep) - logger := log.New() - - // ---- Phase 1: keys destined for files. Each key gets its own - // txNum so per-tx history entries are well-formed. - phase1Keys := genKeys(phase1Seed, keysetSize, keySize) - - { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - - domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) - require.NoError(b, err) - - val := make([]byte, 8) - for i, k := range phase1Keys { - // One key per txNum, packing every txNum in the - // stepsToFlush * aggStep range exactly once. This guarantees - // every step is full so BuildFiles emits a .kv per step. - txNum := uint64(i) - binary.BigEndian.PutUint64(val, txNum+1) // +1 so all values are non-zero - err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) - require.NoError(b, err) - } - - // ComputeCommitment writes the trie root into the CommitmentDomain - // so a follow-up NewSharedDomains can SeekCommitment without - // erroring "commitment state out of date". - lastTxNum := uint64(keysetSize - 1) - _, err = domains.ComputeCommitment(b.Context(), rwTx, true /*save*/, 0, lastTxNum, "h0-bench", nil) - require.NoError(b, err) - - require.NoError(b, domains.Flush(b.Context(), rwTx)) - domains.Close() - require.NoError(b, rwTx.Commit()) - } - - // Build files for the phase-1 range. - require.NoError(b, agg.BuildFiles(uint64(stepsToFlush)*aggStep)) - - // Prune phase-1 entries out of MDBX so they're file-only. - // PruneSmallBatches may report haveMore=true when batch limits stop - // it short of fully draining; loop until drained or we hit a sane - // safety bound. time.Hour keeps it in "furious" prune mode (large - // per-iteration limit), so this is fast in practice. - for round := 0; round < 32; round++ { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - haveMore, err := rwTx.PruneSmallBatches(b.Context(), time.Hour) - require.NoError(b, err) - require.NoError(b, rwTx.Commit()) - if !haveMore { - break - } - } - - // ---- Phase 2: keys destined for MDBX (no BuildFiles after). - phase2Keys := genKeys(phase2Seed, keysetSize, keySize) - - { - rwTx, err := db.BeginTemporalRw(b.Context()) - require.NoError(b, err) - - domains, err := execctx.NewSharedDomains(b.Context(), rwTx, logger) - require.NoError(b, err) - - val := make([]byte, 8) - for i, k := range phase2Keys { - // Phase-2 keys live well past the built-step boundary so a - // hypothetical follow-up BuildFiles wouldn't sweep them in. - txNum := phase2BaseTxNum + uint64(i) - binary.BigEndian.PutUint64(val, txNum+1) - err := domains.DomainPut(domain, rwTx, k, val, txNum, nil) - require.NoError(b, err) - } - - require.NoError(b, domains.Flush(b.Context(), rwTx)) - domains.Close() - require.NoError(b, rwTx.Commit()) - } - - // ---- Open a RO tx for benching. - tx, err := db.BeginTemporalRo(b.Context()) - require.NoError(b, err) - b.Cleanup(func() { tx.Rollback() }) - - aggTx, ok := tx.AggTx().(*state.AggregatorRoTx) - require.True(b, ok, "tx.AggTx() must be *state.AggregatorRoTx") - - // Partition each phase set by actual residency. We expect: - // phase 1 -> almost all "file", with a tail of tip-step keys still - // in MDBX because Prune retains the most-recent step; - // phase 2 -> all "mdbx" (no BuildFiles called for it). - // The bench uses only the keys that landed where we want, so any - // misroute just shrinks the keyset rather than corrupting timings. - mdbxFromP2, fileFromP2, missingP2 := partitionByResidency(b, aggTx, tx, domain, phase2Keys) - mdbxFromP1, fileFromP1, missingP1 := partitionByResidency(b, aggTx, tx, domain, phase1Keys) - if missingP1 > 0 || missingP2 > 0 { - b.Fatalf("residency partition: missing keys p1=%d p2=%d", missingP1, missingP2) - } - b.Logf("residency: phase1 file=%d mdbx=%d phase2 file=%d mdbx=%d", - len(fileFromP1), len(mdbxFromP1), len(fileFromP2), len(mdbxFromP2)) - - mdbxKeys := mdbxFromP2 // bench's mdbx-resident set - fileKeys := fileFromP1 // bench's file-resident set - require.NotEmpty(b, mdbxKeys, "synthetic fixture produced no MDBX-resident keys") - require.NotEmpty(b, fileKeys, "synthetic fixture produced no file-resident keys") - - return &snapBenchSetup{ - tx: tx, - aggTx: aggTx, - domain: domain, - mdbxKeys: mdbxKeys, - fileKeys: fileKeys, - } -} - -// genKeys produces n deterministic keys of size keySize bytes from -// the given seed. Same seed -> same keys -> reproducible benches. -func genKeys(seed uint64, n, keySize int) [][]byte { - rnd := newRnd(seed) - keys := make([][]byte, n) - for i := range keys { - k := make([]byte, keySize) - _, _ = rnd.Read(k) - keys[i] = k - } - return keys -} - -// partitionByResidency walks keys and groups them by which read path -// returns the value: "mdbx" (DebugGetLatestFromDB ok), "file" -// (DebugGetLatestFromDB miss + DebugGetLatestFromFiles ok), or -// "missing" (neither). missing is reported as a count for the caller -// to fail on — no key in our synthetic setup should be unreachable. -func partitionByResidency(b *testing.B, aggTx *state.AggregatorRoTx, tx kv.TemporalTx, domain kv.Domain, keys [][]byte) (mdbxKeys, fileKeys [][]byte, missing int) { - b.Helper() - for _, k := range keys { - _, _, dbOk, err := aggTx.DebugGetLatestFromDB(domain, k, tx) - require.NoError(b, err) - if dbOk { - mdbxKeys = append(mdbxKeys, k) - continue - } - _, fileOk, _, _, err := aggTx.DebugGetLatestFromFiles(domain, k, 0) - require.NoError(b, err) - if fileOk { - fileKeys = append(fileKeys, k) - continue - } - missing++ - } - return -} - -// warmup reads every key in keys via tx.GetLatest so the OS page cache -// is hot for all relevant snapshot/MDBX pages before timing starts. -// Required because the H0 claim only holds for warm pages — we want to -// measure software overhead, not disk I/O. -func warmup(tx kv.TemporalTx, domain kv.Domain, keys [][]byte) { - for _, k := range keys { - _, _, _ = tx.GetLatest(domain, k) - } -} - -// skipIfEmpty short-circuits the calling sub-bench when the key set -// is empty (real-datadir mode often has no MDBX-resident keys after -// aggressive pruning). -func skipIfEmpty(b *testing.B, name string, keys [][]byte) bool { - if len(keys) == 0 { - b.Skipf("no keys for %s sub-bench", name) - return true - } - return false -} - -// runH0 is the H0 measurement body. Four sub-benches: -// -// - MDBX_path: tx.GetLatest with key in MDBX. Hits getLatestFromDb -// and returns; never touches files. Baseline cost. -// -// - File_path: tx.GetLatest with key only in files. Misses MDBX, -// then walks getLatestFromFiles (Bloom -> recsplit -> seg -// decompress). Baseline-with-files cost. -// -// - Forced_file_path: same MDBX-resident key set as MDBX_path but -// routed through DebugGetLatestFromFiles, forcing the file path -// even though the key would have hit MDBX. Isolates per-key -// file-path cost from key-distribution effects. -// -// - Forced_db_path: DebugGetLatestFromDB on the MDBX key set — -// mirror of Forced_file_path, isolates the MDBX-only path cost. -// -// Headline: file_ns_per_op / mdbx_ns_per_op. Forced_* sub-benches -// confirm the ratio isn't a key-set artifact. -func runH0(b *testing.B, setup *snapBenchSetup) { - b.Helper() - - // Two warmup passes: first lands pages in the OS page cache, - // second absorbs L3/TLB effects so we time steady-state. - warmup(setup.tx, setup.domain, setup.mdbxKeys) - warmup(setup.tx, setup.domain, setup.fileKeys) - warmup(setup.tx, setup.domain, setup.mdbxKeys) - warmup(setup.tx, setup.domain, setup.fileKeys) - - b.Run("MDBX_path", func(b *testing.B) { - if skipIfEmpty(b, "MDBX_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)]) - } - }) - - b.Run("File_path", func(b *testing.B) { - if skipIfEmpty(b, "File_path", setup.fileKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) - } - }) - - // Forced_file_path: file-resident keys via the file-only debug path. - // Strips MDBX-miss cost from File_path so we see pure file-side - // work (Bloom -> recsplit -> seg decompress). - b.Run("Forced_file_path", func(b *testing.B) { - if skipIfEmpty(b, "Forced_file_path", setup.fileKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( - setup.domain, setup.fileKeys[i%len(setup.fileKeys)], 0) - } - }) - - // Forced_db_path: MDBX-resident keys via the DB-only debug path. - // Strips routing/dispatch overhead from MDBX_path so we see pure - // MDBX cursor work. - b.Run("Forced_db_path", func(b *testing.B) { - if skipIfEmpty(b, "Forced_db_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _ = setup.aggTx.DebugGetLatestFromDB( - setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], setup.tx) - } - }) - - // Bloom_miss_path: MDBX-resident keys against the file-only debug - // path. These keys are NOT in any .kv file so each lookup is a - // pure xorfilter "not present" probe per file. Gives the Bloom-miss - // cost in isolation — useful for H1 (per-file Bloom probe overhead). - b.Run("Bloom_miss_path", func(b *testing.B) { - if skipIfEmpty(b, "Bloom_miss_path", setup.mdbxKeys) { - return - } - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _, _, _ = setup.aggTx.DebugGetLatestFromFiles( - setup.domain, setup.mdbxKeys[i%len(setup.mdbxKeys)], 0) - } - }) -} - -// BenchmarkSnapVsMDBX_H0_Accounts measures the warm-cache file-vs-MDBX -// per-key gap on the AccountsDomain. Headline: file_ns/op : mdbx_ns/op. -func BenchmarkSnapVsMDBX_H0_Accounts(b *testing.B) { - runH0(b, openSnapBench(b, kv.AccountsDomain)) -} - -// runHGetAsOf measures HistorySeek-via-GetAsOf cost on file-resident -// keys. The motivation (per H0 finding): `getLatestFromFiles` is fast -// (~30 ns isolated); the bloat-workload pprof showed real file-read -// pressure that this can't account for. The calculator's -// HistoryStateReader.Read calls tx.GetAsOf, which goes through -// HistorySeek and walks .ef history files — a different code path -// than getLatestFromFiles. -// -// Sub-benches: -// -// - GetLatest_baseline: tx.GetLatest on file-resident keys; mirrors -// H0's File_path so we can confirm the same setup against a fresh -// comparator. -// -// - GetAsOf_recent: tx.GetAsOf at asOfTxNum = endTxNum - 1. Most -// keys won't have a history record post-asOf, so HistorySeek -// short-circuits to the latest path. Lower bound on GetAsOf cost. -// -// - GetAsOf_mid: tx.GetAsOf at asOfTxNum = endTxNum / 2. Forces -// the .ef walk to seek through more history. This is the cost -// shape the calculator pays when reading historic state. -// -// - GetAsOf_zero: tx.GetAsOf at asOfTxNum = HistoryStartFrom() + 1 -// (just past the visible window start). Maximally adversarial — -// full .ef walk depth. The calculator's old asOfReader.txNum=0 -// bug (PR #21010) hit this path when the window started past 0. -// -// All sub-benches use the same fileKeys set so per-key cost is -// directly comparable across the four. -func runHGetAsOf(b *testing.B, setup *snapBenchSetup) { - b.Helper() - if skipIfEmpty(b, "H_GetAsOf", setup.fileKeys) { - return - } - - // Snapshot the visible-history window so all four sub-benches use - // the same anchors and so we report them in the bench log. We - // don't probe historyStartFrom from here (no public accessor on - // AggregatorRoTx for per-domain history range); instead use simple - // fixed fractions of endTxNum. Adjust asOfFloor=1 to avoid - // hitting txNum=0 which can error on snapshot-loaded chains - // (PR #21010 was that bug; we don't want H_GetAsOf to trip on it). - endTxNum := setup.aggTx.EndTxNumNoCommitment() - asOfRecent := endTxNum - if endTxNum > 0 { - asOfRecent = endTxNum - 1 - } - asOfMid := endTxNum / 2 - asOfFloor := uint64(1) - b.Logf("H_GetAsOf anchors: endTxNum=%d -> asOfRecent=%d asOfMid=%d asOfFloor=%d", - endTxNum, asOfRecent, asOfMid, asOfFloor) - - // Two warmup passes so the OS page cache holds the .ef files we - // care about. Touch each anchor txNum so the relevant history - // segments are mmap'd in. - for _, asOf := range []uint64{asOfRecent, asOfMid, asOfFloor} { - for _, k := range setup.fileKeys { - _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) - } - for _, k := range setup.fileKeys { - _, _, _ = setup.tx.GetAsOf(setup.domain, k, asOf) - } - } - - b.Run("GetLatest_baseline", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetLatest(setup.domain, setup.fileKeys[i%len(setup.fileKeys)]) - } - }) - - b.Run("GetAsOf_recent", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfRecent) - } - }) - - b.Run("GetAsOf_mid", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfMid) - } - }) - - b.Run("GetAsOf_floor", func(b *testing.B) { - b.ReportAllocs() - for i := 0; b.Loop(); i++ { - _, _, _ = setup.tx.GetAsOf(setup.domain, setup.fileKeys[i%len(setup.fileKeys)], asOfFloor) - } - }) -} - -// BenchmarkSnapVsMDBX_HGetAsOf_Accounts measures HistorySeek cost on -// real file-resident keys via tx.GetAsOf. Confirms the H0 finding -// that the production bloat-workload bottleneck is in .ef history -// walking, not .kv latest-state reads. -func BenchmarkSnapVsMDBX_HGetAsOf_Accounts(b *testing.B) { - runHGetAsOf(b, openSnapBench(b, kv.AccountsDomain)) -} - -// BenchmarkSnapVsMDBX_HGetAsOf_Storage — same for StorageDomain. -// Storage tends to dominate in the bloat workload. -func BenchmarkSnapVsMDBX_HGetAsOf_Storage(b *testing.B) { - runHGetAsOf(b, openSnapBench(b, kv.StorageDomain)) -} - -// BenchmarkSnapVsMDBX_H0_Storage — same harness for StorageDomain. -// Storage dominates file-read traffic on bloat workloads (per -// runs-step9-cache-behind-sd memory), so getting the gap for both -// Accounts and Storage is the minimum useful H0 output. -func BenchmarkSnapVsMDBX_H0_Storage(b *testing.B) { - runH0(b, openSnapBench(b, kv.StorageDomain)) -} diff --git a/execution/commitment/hex_concurrent_patricia_hashed.go b/execution/commitment/hex_concurrent_patricia_hashed.go index 7d5424b5df8..bbac1a6d2c3 100644 --- a/execution/commitment/hex_concurrent_patricia_hashed.go +++ b/execution/commitment/hex_concurrent_patricia_hashed.go @@ -165,6 +165,7 @@ func (p *ConcurrentPatriciaHashed) SetTraceDomain(b bool) { p.mounts[i].SetTraceDomain(b) } } + // SetBranchCache attaches the same BranchCache instance to the root trie // and all 16 mounts. Sharing one cache across mounts is correct under the // concurrency contract (branch_cache.go): mounts partition the prefix diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index eedacefab08..060057af560 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2896,8 +2896,9 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log return rootHash, nil } -func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } -func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } +func (hph *HexPatriciaHashed) SetTrace(trace bool) { hph.trace = trace } +func (hph *HexPatriciaHashed) SetTraceDomain(trace bool) { hph.traceDomain = trace } + // SetBranchCache attaches a BranchCache for branch read-through and // write-through. Also propagates to the trie's BranchEncoder so encoder // writes update the cache via mark-dirty-then-Put (see CollectUpdate). From fa4d7f7b5db3bfbdb072f912745181c93afdb94b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 16:42:49 +0000 Subject: [PATCH 061/120] perf(r3): bulk storage-trunk preload via two-parity range scan (core + tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-prefix GetLatest BFS with a sequential range scan: a contract H's storage-trunk branches live in two CommitmentDomain key ranges (split by HexToCompact's HP-flag parity), so contractTrunkKeyRanges computes [0x00||H, nextSubtree(0x00||H)) for even-depth branches and [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))) for odd-depth, and LoadBulk drains both via a CommitmentRangeReader, collects every branch of the subtree, sorts shallowest-first (the BFS/highest-reuse ordering — a lexicographic scan would be depth-first within a subtree, which under a budget pins a deep narrow slice), and pins from the front until the byte budget is hit. Re-scanning on a later LoadBulk extends the pinned set (resumable/phased, like Run). This commit is the algorithm + ContractTrunkPreload.LoadBulk + PreloadContractTrunkBulk + unit tests (range computation, nextSubtree, shallowest-first-within-budget, phased extend). The per-prefix Run/BFS path is left intact; wiring triggerTrunkPreload to provide the real CommitmentRangeReader (aggTx.DebugRangeLatestFromFiles) and call LoadBulk is the next commit. Refs perf-research-tracker.md R3. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload.go | 2 + execution/commitment/preload_bulk.go | 195 +++++++++++++++++ execution/commitment/preload_bulk_test.go | 243 ++++++++++++++++++++++ 3 files changed, 440 insertions(+) create mode 100644 execution/commitment/preload_bulk.go create mode 100644 execution/commitment/preload_bulk_test.go diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..73c3b09a0da 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -49,6 +49,7 @@ type pathDepth struct { // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte + contractNibbles []byte // 64 nibbles of contractHash, cached for the bulk range scan (preload_bulk.go) queue []pathDepth pinnedPrefixes [][]byte pinned int @@ -70,6 +71,7 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) } return &ContractTrunkPreload{ contractHash: contractHash, + contractNibbles: contractNibbles, queue: []pathDepth{{path: contractNibbles, depth: 64}}, maxDepthReached: 64, }, nil diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go new file mode 100644 index 00000000000..34d1540e009 --- /dev/null +++ b/execution/commitment/preload_bulk.go @@ -0,0 +1,195 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "bytes" + "fmt" + "sort" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// CommitmentRangeReader yields, in ascending key order, every latest +// CommitmentDomain entry whose key K satisfies fromKey <= K < toKey. yield +// returns false to stop early. step is the domain step the value came from +// (forwarded to BranchCache.PinEntry). Decoupled (callback form) so the +// commitment package doesn't import db/kv/stream. +type CommitmentRangeReader func(fromKey, toKey []byte, yield func(key, value []byte, step uint64) bool) error + +// nextSubtree returns the smallest key K' such that K' is greater than every +// key having `in` as a prefix (i.e. the exclusive upper bound for a +// prefix-range scan over `in`). Returns nil if `in` is all 0xff (no such K'). +// Equivalent to db/kv.NextSubtree; inlined to keep this package's import set +// minimal. +func nextSubtree(in []byte) []byte { + r := make([]byte, len(in)) + copy(r, in) + for i := len(r) - 1; i >= 0; i-- { + if r[i] != 0xff { + r[i]++ + return r[:i+1] + } + } + return nil // all 0xff +} + +// contractTrunkKeyRanges returns the two CommitmentDomain key ranges that +// together hold every branch node of a contract's storage subtree. +// +// The commitment domain keys branches by HexToCompact(nibblePath). A storage +// branch of contract H (32 bytes => 64 nibbles) at slot-path of k nibbles has +// nibblePath = H_nibbles ++ slotNibbles, total 64+k nibbles. HexToCompact's +// HP flag byte differs by parity of the total length, so the branches split +// into two non-adjacent byte ranges: +// +// even total length (k even — incl. the depth-64 subtree-root branch): +// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) +// odd total length (k odd): +// key = (0x10 | H[0]>>4) || <31 bytes derived from H[1:]> || +// || ... +// ⇒ all such keys lie in [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))). +// +// (HexToCompact of the 64-nibble path = 0x00||H; of a 65-nibble path = the +// 33-byte odd prefix with the final byte's low nibble = the 65th nibble.) +func contractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { + evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes + evenTo = nextSubtree(evenFrom) + + odd0 := make([]byte, 0, 65) + odd0 = append(append(odd0, contractNibbles...), 0) + oddF := make([]byte, 0, 65) + oddF = append(append(oddF, contractNibbles...), 15) + oddFrom = nibbles.HexToCompact(odd0) + oddTo = nextSubtree(nibbles.HexToCompact(oddF)) + return evenFrom, evenTo, oddFrom, oddTo +} + +// maxStorageTrunkDepth is the deepest total nibble path a storage branch can +// have: 64 (account path) + 64 (keccak256(slot)) = 128. Used as a hard sanity +// cap on what the bulk scan considers; the byte budget is the real bound. +const maxStorageTrunkDepth = 128 + +type bulkBranch struct { + key []byte + value []byte + step uint64 + depth int // len(nibblePath) +} + +// LoadBulk pins this contract's storage-trunk branches into cache by scanning +// the two CommitmentDomain key ranges (one sequential file+DB-merged range scan +// per parity, via rr) instead of N per-prefix point lookups. It collects every +// branch of the contract's subtree, sorts them shallowest-first (the +// BFS/highest-reuse-first ordering — a lexicographic scan would yield +// depth-first-within-subtree order, which under a budget pins a deep narrow +// slice), and pins from the front until budgetBytes is reached. +// +// Returns the number of branches pinned by this call (cumulative total via +// PinnedTotal). Calling LoadBulk again with a larger total budget extends the +// pinned set: it re-scans (a sequential range scan is cheap), skips the +// already-pinned shallowest entries, and pins the next budget's worth — the +// resumable/phased semantics of Run, implemented by re-scan. +func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (newlyPinned int, err error) { + if cache == nil { + return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: cache is nil") + } + if rr == nil { + return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: range reader is nil") + } + if budgetBytes <= p.usedBytes { + return 0, nil + } + + evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(p.contractNibbles) + + var collected []bulkBranch + collect := func(from, to []byte) error { + return rr(from, to, func(key, value []byte, step uint64) bool { + path := nibbles.CompactToHex(key) + if len(path) < 64 || len(path) > maxStorageTrunkDepth || !bytes.Equal(path[:64], p.contractNibbles) { + return true // not a branch of this contract's subtree — skip, keep scanning + } + kc := make([]byte, len(key)) + copy(kc, key) + vc := make([]byte, len(value)) + copy(vc, value) + collected = append(collected, bulkBranch{key: kc, value: vc, step: step, depth: len(path)}) + return true + }) + } + if err = collect(evenFrom, evenTo); err != nil { + return 0, fmt.Errorf("LoadBulk even-range scan: %w", err) + } + if err = collect(oddFrom, oddTo); err != nil { + return 0, fmt.Errorf("LoadBulk odd-range scan: %w", err) + } + + // Shallowest first; tie-break by path bytes for determinism. + sort.Slice(collected, func(i, j int) bool { + if collected[i].depth != collected[j].depth { + return collected[i].depth < collected[j].depth + } + return bytes.Compare(collected[i].key, collected[j].key) < 0 + }) + + alreadyPinned := p.pinned // skip the entries a previous LoadBulk already pinned (same shallowest-first order) + chunkPinned := 0 + chunkBytes := 0 + for idx, b := range collected { + if idx < alreadyPinned { + continue + } + entryCost := estimatedEntryOverheadBytes + len(b.key) + len(b.value) + if p.usedBytes+chunkBytes+entryCost > budgetBytes { + break + } + cache.PinEntry(b.key, b.value, b.step, "preload-trunk-bulk") + p.pinnedPrefixes = append(p.pinnedPrefixes, b.key) + chunkBytes += entryCost + chunkPinned++ + if b.depth > p.maxDepthReached { + p.maxDepthReached = b.depth + } + if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { + logger.Info("[trunk-preload-bulk] progress", + "pinned", p.pinned+chunkPinned, "depth", b.depth, "used_mb", (p.usedBytes+chunkBytes)/(1<<20)) + } + } + p.pinned += chunkPinned + p.usedBytes += chunkBytes + if logger != nil { + logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), + "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, + "used_mb", p.usedBytes/(1<<20), "max_depth_reached", p.maxDepthReached, "budget_exhausted", p.pinned < len(collected)) + } + return chunkPinned, nil +} + +// PreloadContractTrunkBulk is the bulk analogue of PreloadContractTrunk: one +// call, bounded by ramBudgetBytes, using a range scan. +func PreloadContractTrunkBulk(contractHash []byte, ramBudgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (int, error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkBulk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreload(contractHash) + if err != nil { + return 0, err + } + return p.LoadBulk(ramBudgetBytes, rr, cache, logger) +} diff --git a/execution/commitment/preload_bulk_test.go b/execution/commitment/preload_bulk_test.go new file mode 100644 index 00000000000..4b0ab650ac4 --- /dev/null +++ b/execution/commitment/preload_bulk_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. + +package commitment + +import ( + "bytes" + "encoding/binary" + "sort" + "testing" + + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +func hexNibbles(b []byte) []byte { + out := make([]byte, len(b)*2) + for i, x := range b { + out[2*i] = x >> 4 + out[2*i+1] = x & 0x0f + } + return out +} + +// keyForBranch builds the CommitmentDomain key for a branch of contract H's +// storage subtree reached by the given slot-path nibbles. +func keyForBranch(contractNibbles, slotPath []byte) []byte { + full := make([]byte, 0, len(contractNibbles)+len(slotPath)) + full = append(full, contractNibbles...) + full = append(full, slotPath...) + return nibbles.HexToCompact(full) +} + +func inRange(k, from, to []byte) bool { + return bytes.Compare(k, from) >= 0 && (to == nil || bytes.Compare(k, to) < 0) +} + +func TestContractTrunkKeyRanges(t *testing.T) { + hashA := make([]byte, 32) + for i := range hashA { + hashA[i] = byte(7*i + 3) // arbitrary + } + hashB := make([]byte, 32) + for i := range hashB { + hashB[i] = byte(251 - 3*i) + } + nibA := hexNibbles(hashA) + nibB := hexNibbles(hashB) + + evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(nibA) + + // Every branch of A at assorted slot-paths must land in exactly one range, + // per the parity of its total nibble length. + slotPaths := [][]byte{ + {}, // depth 64 — subtree root + {0x0}, // 65 + {0xf}, // 65 + {0x1, 0x2}, // 66 + {0xf, 0xf}, // 66 + {0x3, 0x4, 0x5}, // 67 + {0x6, 0x7, 0x8, 0x9}, // 68 + {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 + make([]byte, 64), // 128 — deepest + } + for _, sp := range slotPaths { + k := keyForBranch(nibA, sp) + total := 64 + len(sp) + if total%2 == 0 { + if !inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d branch %x not in even range [%x,%x)", total, k, evenFrom, evenTo) + } + if inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d (even) branch %x unexpectedly in odd range", total, k) + } + } else { + if !inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d branch %x not in odd range [%x,%x)", total, k, oddFrom, oddTo) + } + if inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d (odd) branch %x unexpectedly in even range", total, k) + } + } + // round-trip: CompactToHex(key) must reproduce the full path + got := nibbles.CompactToHex(k) + want := append(append([]byte{}, nibA...), sp...) + if !bytes.Equal(got, want) { + t.Fatalf("CompactToHex round-trip mismatch: got %x want %x", got, want) + } + } + + // A different contract's branches must be in neither of A's ranges. + for _, sp := range slotPaths[:6] { + k := keyForBranch(nibB, sp) + if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) + } + } +} + +func TestNextSubtree(t *testing.T) { + cases := []struct{ in, want []byte }{ + {[]byte{0x01, 0x02}, []byte{0x01, 0x03}}, + {[]byte{0x01, 0xff}, []byte{0x02}}, + {[]byte{0x00}, []byte{0x01}}, + } + for _, c := range cases { + if got := nextSubtree(c.in); !bytes.Equal(got, c.want) { + t.Fatalf("nextSubtree(%x) = %x, want %x", c.in, got, c.want) + } + } + if nextSubtree([]byte{0xff, 0xff}) != nil { + t.Fatalf("nextSubtree(0xffff) should be nil") + } +} + +// branchVal returns a deterministic branch-node value of size sz with a valid +// 4-byte header (touchMap||afterMap) — afterMap=0 so the BFS-children logic +// (in Run, not LoadBulk) would queue nothing; LoadBulk ignores the bitmap. +func branchVal(seed, sz int) []byte { + if sz < 4 { + sz = 4 + } + v := make([]byte, sz) + binary.BigEndian.PutUint16(v[0:2], uint16(seed)) + for i := 4; i < sz; i++ { + v[i] = byte(seed + i) + } + return v +} + +func TestLoadBulk_ShallowestFirstWithinBudget(t *testing.T) { + hash := make([]byte, 32) + for i := range hash { + hash[i] = byte(0x40 + i) + } + nib := hexNibbles(hash) + + // Synthetic branch set at depths 64..69, mixed parity, fixed value size. + const valSz = 100 + type ent struct { + key []byte + val []byte + depth int + } + var ents []ent + add := func(sp []byte) { + ents = append(ents, ent{key: keyForBranch(nib, sp), val: branchVal(len(ents)+1, valSz), depth: 64 + len(sp)}) + } + add([]byte{}) // 64 + add([]byte{0x1}) // 65 + add([]byte{0x2}) // 65 + add([]byte{0x3, 0x4}) // 66 + add([]byte{0x5, 0x6}) // 66 + add([]byte{0x7, 0x8, 0x9}) // 67 + add([]byte{0xa, 0xb, 0xc, 0xd}) // 68 + add([]byte{0xe, 0xf, 0x0, 0x1, 0x2}) // 69 + + // Fake range reader: yields the entries whose key falls in [from,to), in + // ascending key order. + rr := CommitmentRangeReader(func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + var in []ent + for _, e := range ents { + if bytes.Compare(e.key, from) >= 0 && (to == nil || bytes.Compare(e.key, to) < 0) { + in = append(in, e) + } + } + sort.Slice(in, func(i, j int) bool { return bytes.Compare(in[i].key, in[j].key) < 0 }) + for _, e := range in { + if !yield(e.key, e.val, 1) { + return nil + } + } + return nil + }) + + entryCost := estimatedEntryOverheadBytes + 33 /*≈compact key len*/ + valSz // approx; key lens are 33-36 + // budget for ~the 4 shallowest (depths 64,65,65,66) — give a bit of slack + budget := 4*entryCost + 80 + + c := NewBranchCache(100) + p, err := NewContractTrunkPreload(hash) + if err != nil { + t.Fatal(err) + } + n, err := p.LoadBulk(budget, rr, c, nil) + if err != nil { + t.Fatal(err) + } + if n == 0 || n != p.PinnedTotal() { + t.Fatalf("LoadBulk pinned %d (PinnedTotal=%d), expected >0 and consistent", n, p.PinnedTotal()) + } + if c.PinnedCount() != n { + t.Fatalf("cache PinnedCount=%d != pinned %d", c.PinnedCount(), n) + } + // Sort the synthetic set the way LoadBulk does (shallowest first, key tie-break). + sortedExpected := append([]ent(nil), ents...) + sort.Slice(sortedExpected, func(i, j int) bool { + if sortedExpected[i].depth != sortedExpected[j].depth { + return sortedExpected[i].depth < sortedExpected[j].depth + } + return bytes.Compare(sortedExpected[i].key, sortedExpected[j].key) < 0 + }) + // The first n pinned must be exactly the n shallowest. + for i := 0; i < n; i++ { + v, _, ok := c.Get(sortedExpected[i].key) + if !ok { + t.Fatalf("expected branch #%d (depth %d, key %x) to be pinned, but it's not in cache", i, sortedExpected[i].depth, sortedExpected[i].key) + } + if !bytes.Equal(v, sortedExpected[i].val) { + t.Fatalf("pinned branch #%d value mismatch", i) + } + } + // The (n+1)-th shallowest must NOT be pinned (budget cut it off). + if n < len(ents) { + if _, _, ok := c.Get(sortedExpected[n].key); ok { + t.Fatalf("branch #%d should have been excluded by the budget but is pinned", n) + } + } + if p.MaxDepthReached() != sortedExpected[n-1].depth { + t.Fatalf("MaxDepthReached=%d, expected %d", p.MaxDepthReached(), sortedExpected[n-1].depth) + } + + // Phased extend: a larger total budget pins the rest. + n2, err := p.LoadBulk(1<<20, rr, c, nil) + if err != nil { + t.Fatal(err) + } + if n2 == 0 || p.PinnedTotal() != len(ents) { + t.Fatalf("extend: pinned %d more, total %d, expected total %d", n2, p.PinnedTotal(), len(ents)) + } + if c.PinnedCount() != len(ents) { + t.Fatalf("after extend, cache PinnedCount=%d != %d", c.PinnedCount(), len(ents)) + } +} From b4591a25312e9651397b5866537fbd840d4be275 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 16:58:26 +0000 Subject: [PATCH 062/120] perf(r3): wire triggerTrunkPreload to the bulk range-scan path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit triggerTrunkPreload's goroutine now builds a CommitmentRangeReader over tx.RangeLatest(CommitmentDomain, from, to) and calls PreloadContractTrunkBulk; PIN_TRUNK_BULK=false reverts to the per-prefix GetLatest BFS (PreloadContractTrunk). step is passed as 0 (the pinned-tier step is a coherency hint, corrected in-place by a later same-prefix write; the bulk scan reads the latest committed value). Refs perf-research-tracker.md R3. Adaptive-controller path (Run-based extend) still uses the per-prefix reader — to be migrated to LoadBulk in a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 34 +++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index da2b9a97610..7b1f7dc8f29 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1368,10 +1368,40 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } return v, uint64(step), len(v) > 0, nil } + // R3: bulk range scan of the contract's two CommitmentDomain key + // ranges (one sequential file+DB-merged scan per parity) instead of + // the per-prefix GetLatest BFS — ~N random cold seeks become a couple + // of sequential cold reads. step is 0 here: the pinned-tier step is a + // coherency hint (a later same-prefix write at step>0 updates the + // entry in-place), and the bulk scan reads the latest committed value, + // so 0 is conservative-correct. PIN_TRUNK_BULK=false → BFS fallback. + useBulk := dbg.EnvBool("PIN_TRUNK_BULK", true) + rr := func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + it, err := tx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) + if err != nil { + return err + } + defer it.Close() + for it.HasNext() { + k, v, err := it.Next() + if err != nil { + return err + } + if !yield(k, v, 0) { + return nil + } + } + return nil + } for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "bulk", useBulk) + var n int + if useBulk { + n, err = commitment.PreloadContractTrunkBulk(h, ramBudgetBytes, rr, branchCache, logger) + } else { + n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + } took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", From ae86359a2cc3e307ca37b22d1bbbdcaaf5efff2f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 11 May 2026 17:08:54 +0000 Subject: [PATCH 063/120] perf(r3): adaptive controller uses LoadBulk (range scan) when available AdaptivePinController.OnBlockComplete now takes both a CommitmentReader (per-prefix BFS) and an optional CommitmentRangeReader (bulk range scan); initial-view promotes and per-block extends use LoadBulk when rr != nil, else fall back to Run. domain_shared.go's Flush-time hook builds rr over tx.Debug().RangeLatest(CommitmentDomain, ...) when PIN_TRUNK_BULK (default true) and passes it. Adds ContractTrunkPreload.HasMore() (BFS queue non-empty OR last bulk scan was budget-limited) so the extend gate works for both paths; lastBulkHadMore tracked in LoadBulk. Test OnBlockComplete calls pass nil for rr (exercise the controller policy on the BFS path; LoadBulk itself is unit-tested separately). Refs perf-research-tracker.md R3. Remaining: bench validation on the SSTORE-bloat fixture (needs perf-devnet-3 datadir re-downloaded). Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 22 +++++++++++++- execution/commitment/adaptive_pin.go | 41 +++++++++++++++++++------- execution/commitment/preload.go | 5 ++++ execution/commitment/preload_bulk.go | 1 + execution/commitment/trunk_pin_test.go | 8 ++--- 5 files changed, 61 insertions(+), 16 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 7b1f7dc8f29..0c53500faed 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,7 +783,27 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + var rr commitment.CommitmentRangeReader + if dbg.EnvBool("PIN_TRUNK_BULK", true) { + rr = func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { + it, err := ttx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) + if err != nil { + return err + } + defer it.Close() + for it.HasNext() { + k, v, err := it.Next() + if err != nil { + return err + } + if !yield(k, v, 0) { + return nil + } + } + return nil + } + } + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader, rr) } } // Refresh pinned-tier gauges once per Flush — once-per-batch diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 1cd6b6f6066..a0bd2249bf1 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -154,23 +154,42 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). The reader is the CommitmentReader -// for the just-committed state, used by initial-view preload and -// per-block extension. +// boundaries (after SD.Flush). +// +// If rr (a CommitmentRangeReader) is non-nil, initial-view preloads and +// extensions use the bulk range scan (LoadBulk); otherwise they fall back +// to the per-prefix BFS via reader (a CommitmentReader). The host passes +// rr when PIN_TRUNK_BULK is enabled (default). At least one of the two +// must be non-nil. // // Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run -// inline; sized so per-block work fits within typical inter-block -// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). +// is available for the NEXT block's reads. Extensions also run inline. // // Logs a [adaptive-pin] line per block when any state changes or // promoted contracts exist. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader, rr CommitmentRangeReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() + // load runs an initial-view (totalBudget = budget) or extension + // (totalBudget = used + step) preload via the bulk path if available, + // else the per-prefix BFS. + load := func(p *ContractTrunkPreload, totalBudget int) error { + if rr != nil { + _, err := p.LoadBulk(totalBudget, rr, c.cache, c.logger) + return err + } + // BFS Run takes the *additional* budget; map total → additional. + add := totalBudget - p.UsedBytes() + if add <= 0 { + return nil + } + _, _, err := p.Run(add, reader, c.cache, c.logger) + return err + } + var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -179,14 +198,14 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and queue not empty. - if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + // Extend if budget remains and more branches are pending. + if state.preload.HasMore() && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + if err := load(state.preload, state.preload.UsedBytes()+step); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -212,7 +231,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if err != nil { continue } - if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + if err := load(p, c.cfg.InitialViewBudgetBytes); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) // Roll back partial pin so we don't leak entries // for a contract that won't be in c.states. diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 73c3b09a0da..cc449d9ba0a 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -55,6 +55,7 @@ type ContractTrunkPreload struct { pinned int usedBytes int maxDepthReached int + lastBulkHadMore bool // last LoadBulk hit the byte budget with branches left unpinned } // NewContractTrunkPreload constructs a preload state seeded at depth 64 @@ -181,6 +182,10 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } +// HasMore reports whether more branches remain to be pinned (BFS queue +// non-empty, or the last bulk scan was cut short by the byte budget). +func (p *ContractTrunkPreload) HasMore() bool { return len(p.queue) > 0 || p.lastBulkHadMore } + // PinnedPrefixes returns the prefix slices this preload has added to // the cache so far. The adaptive controller uses this to Invalidate // the contract's pin set on demotion. The returned slice aliases diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go index 34d1540e009..6d4f0927df9 100644 --- a/execution/commitment/preload_bulk.go +++ b/execution/commitment/preload_bulk.go @@ -173,6 +173,7 @@ func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReade } p.pinned += chunkPinned p.usedBytes += chunkBytes + p.lastBulkHadMore = p.pinned < len(collected) if logger != nil { logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index edf8ebac6b0..588d86de649 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -189,7 +189,7 @@ func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { } // OnBlockComplete should promote the contract. - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1) require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) require.Greater(t, c.PinnedCount(), 0) @@ -215,17 +215,17 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { // Hot block: triggers promotion. _, _, _ = c.Get(prefix) - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1) pinnedAfterPromote := c.PinnedCount() require.Greater(t, pinnedAfterPromote, 0) // Cold blocks: no misses for the contract. After cooldown the // controller demotes and invalidates the pin set. - ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") - ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false), nil) require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } From 4ca92847db0e0c1699e7a36e7f5eb3fc81ab750a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 15:22:05 +0000 Subject: [PATCH 064/120] perf(preload): retire the broken R3 range-scan preload The R3 'bulk preload via two-parity range scan' (LoadBulk + Debug().RangeLatest + adaptive wiring) regressed the SSTORE-bloat docker bench 13.2 -> 5.6 mgas/s (below the no-pin baseline). Root cause: the commitment domain .kv files are hashmap-indexed (.kvi, no ordinal lookup) and total 515 GB, so RangeLatest over them degrades to a linear decompress-and-skip from byte 0 -- orders of magnitude slower than the 89k targeted point lookups it replaced; LoadBulk also collects->sorts->pins (no incremental benefit, loses the preload-vs-block race); and the debug raw iterator returns un-dereferenced (shortened-key) branch values. Reverts domain_shared.go / adaptive_pin.go / preload.go / trunk_pin_test.go to the trunk-pin BFS baseline; deletes preload_bulk.go + preload_bulk_test.go. The replacement (two-phase parallel preload: bulk DB range-cursor + parallel file-only wave-BFS) is designed in agentspecs/commitment-branch-preload-redesign.md and lands in follow-up commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 56 +---- execution/commitment/adaptive_pin.go | 41 +--- execution/commitment/preload.go | 7 - execution/commitment/preload_bulk.go | 196 ----------------- execution/commitment/preload_bulk_test.go | 243 ---------------------- execution/commitment/trunk_pin_test.go | 8 +- 6 files changed, 18 insertions(+), 533 deletions(-) delete mode 100644 execution/commitment/preload_bulk.go delete mode 100644 execution/commitment/preload_bulk_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 0c53500faed..da2b9a97610 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,27 +783,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } - var rr commitment.CommitmentRangeReader - if dbg.EnvBool("PIN_TRUNK_BULK", true) { - rr = func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - it, err := ttx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) - if err != nil { - return err - } - defer it.Close() - for it.HasNext() { - k, v, err := it.Next() - if err != nil { - return err - } - if !yield(k, v, 0) { - return nil - } - } - return nil - } - } - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader, rr) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) } } // Refresh pinned-tier gauges once per Flush — once-per-batch @@ -1388,40 +1368,10 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach } return v, uint64(step), len(v) > 0, nil } - // R3: bulk range scan of the contract's two CommitmentDomain key - // ranges (one sequential file+DB-merged scan per parity) instead of - // the per-prefix GetLatest BFS — ~N random cold seeks become a couple - // of sequential cold reads. step is 0 here: the pinned-tier step is a - // coherency hint (a later same-prefix write at step>0 updates the - // entry in-place), and the bulk scan reads the latest committed value, - // so 0 is conservative-correct. PIN_TRUNK_BULK=false → BFS fallback. - useBulk := dbg.EnvBool("PIN_TRUNK_BULK", true) - rr := func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - it, err := tx.Debug().RangeLatest(kv.CommitmentDomain, from, to, -1) - if err != nil { - return err - } - defer it.Close() - for it.HasNext() { - k, v, err := it.Next() - if err != nil { - return err - } - if !yield(k, v, 0) { - return nil - } - } - return nil - } for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "bulk", useBulk) - var n int - if useBulk { - n, err = commitment.PreloadContractTrunkBulk(h, ramBudgetBytes, rr, branchCache, logger) - } else { - n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) - } + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) + n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index a0bd2249bf1..1cd6b6f6066 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -154,42 +154,23 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). -// -// If rr (a CommitmentRangeReader) is non-nil, initial-view preloads and -// extensions use the bulk range scan (LoadBulk); otherwise they fall back -// to the per-prefix BFS via reader (a CommitmentReader). The host passes -// rr when PIN_TRUNK_BULK is enabled (default). At least one of the two -// must be non-nil. +// boundaries (after SD.Flush). The reader is the CommitmentReader +// for the just-committed state, used by initial-view preload and +// per-block extension. // // Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run inline. +// is available for the NEXT block's reads. Extensions also run +// inline; sized so per-block work fits within typical inter-block +// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). // // Logs a [adaptive-pin] line per block when any state changes or // promoted contracts exist. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader, rr CommitmentRangeReader) { +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() - // load runs an initial-view (totalBudget = budget) or extension - // (totalBudget = used + step) preload via the bulk path if available, - // else the per-prefix BFS. - load := func(p *ContractTrunkPreload, totalBudget int) error { - if rr != nil { - _, err := p.LoadBulk(totalBudget, rr, c.cache, c.logger) - return err - } - // BFS Run takes the *additional* budget; map total → additional. - add := totalBudget - p.UsedBytes() - if add <= 0 { - return nil - } - _, _, err := p.Run(add, reader, c.cache, c.logger) - return err - } - var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -198,14 +179,14 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and more branches are pending. - if state.preload.HasMore() && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { + // Extend if budget remains and queue not empty. + if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if err := load(state.preload, state.preload.UsedBytes()+step); err != nil { + if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -231,7 +212,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if err != nil { continue } - if err := load(p, c.cfg.InitialViewBudgetBytes); err != nil { + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) // Roll back partial pin so we don't leak entries // for a contract that won't be in c.states. diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index cc449d9ba0a..88fa7cc2677 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -49,13 +49,11 @@ type pathDepth struct { // Caller MUST NOT use the same instance from multiple goroutines. type ContractTrunkPreload struct { contractHash []byte - contractNibbles []byte // 64 nibbles of contractHash, cached for the bulk range scan (preload_bulk.go) queue []pathDepth pinnedPrefixes [][]byte pinned int usedBytes int maxDepthReached int - lastBulkHadMore bool // last LoadBulk hit the byte budget with branches left unpinned } // NewContractTrunkPreload constructs a preload state seeded at depth 64 @@ -72,7 +70,6 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) } return &ContractTrunkPreload{ contractHash: contractHash, - contractNibbles: contractNibbles, queue: []pathDepth{{path: contractNibbles, depth: 64}}, maxDepthReached: 64, }, nil @@ -182,10 +179,6 @@ func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } // MaxDepthReached returns the deepest trie depth pinned so far. func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } -// HasMore reports whether more branches remain to be pinned (BFS queue -// non-empty, or the last bulk scan was cut short by the byte budget). -func (p *ContractTrunkPreload) HasMore() bool { return len(p.queue) > 0 || p.lastBulkHadMore } - // PinnedPrefixes returns the prefix slices this preload has added to // the cache so far. The adaptive controller uses this to Invalidate // the contract's pin set on demotion. The returned slice aliases diff --git a/execution/commitment/preload_bulk.go b/execution/commitment/preload_bulk.go deleted file mode 100644 index 6d4f0927df9..00000000000 --- a/execution/commitment/preload_bulk.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package commitment - -import ( - "bytes" - "fmt" - "sort" - - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/commitment/nibbles" -) - -// CommitmentRangeReader yields, in ascending key order, every latest -// CommitmentDomain entry whose key K satisfies fromKey <= K < toKey. yield -// returns false to stop early. step is the domain step the value came from -// (forwarded to BranchCache.PinEntry). Decoupled (callback form) so the -// commitment package doesn't import db/kv/stream. -type CommitmentRangeReader func(fromKey, toKey []byte, yield func(key, value []byte, step uint64) bool) error - -// nextSubtree returns the smallest key K' such that K' is greater than every -// key having `in` as a prefix (i.e. the exclusive upper bound for a -// prefix-range scan over `in`). Returns nil if `in` is all 0xff (no such K'). -// Equivalent to db/kv.NextSubtree; inlined to keep this package's import set -// minimal. -func nextSubtree(in []byte) []byte { - r := make([]byte, len(in)) - copy(r, in) - for i := len(r) - 1; i >= 0; i-- { - if r[i] != 0xff { - r[i]++ - return r[:i+1] - } - } - return nil // all 0xff -} - -// contractTrunkKeyRanges returns the two CommitmentDomain key ranges that -// together hold every branch node of a contract's storage subtree. -// -// The commitment domain keys branches by HexToCompact(nibblePath). A storage -// branch of contract H (32 bytes => 64 nibbles) at slot-path of k nibbles has -// nibblePath = H_nibbles ++ slotNibbles, total 64+k nibbles. HexToCompact's -// HP flag byte differs by parity of the total length, so the branches split -// into two non-adjacent byte ranges: -// -// even total length (k even — incl. the depth-64 subtree-root branch): -// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) -// odd total length (k odd): -// key = (0x10 | H[0]>>4) || <31 bytes derived from H[1:]> || -// || ... -// ⇒ all such keys lie in [HexToCompact(H_nibbles||0), nextSubtree(HexToCompact(H_nibbles||15))). -// -// (HexToCompact of the 64-nibble path = 0x00||H; of a 65-nibble path = the -// 33-byte odd prefix with the final byte's low nibble = the 65th nibble.) -func contractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { - evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes - evenTo = nextSubtree(evenFrom) - - odd0 := make([]byte, 0, 65) - odd0 = append(append(odd0, contractNibbles...), 0) - oddF := make([]byte, 0, 65) - oddF = append(append(oddF, contractNibbles...), 15) - oddFrom = nibbles.HexToCompact(odd0) - oddTo = nextSubtree(nibbles.HexToCompact(oddF)) - return evenFrom, evenTo, oddFrom, oddTo -} - -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. Used as a hard sanity -// cap on what the bulk scan considers; the byte budget is the real bound. -const maxStorageTrunkDepth = 128 - -type bulkBranch struct { - key []byte - value []byte - step uint64 - depth int // len(nibblePath) -} - -// LoadBulk pins this contract's storage-trunk branches into cache by scanning -// the two CommitmentDomain key ranges (one sequential file+DB-merged range scan -// per parity, via rr) instead of N per-prefix point lookups. It collects every -// branch of the contract's subtree, sorts them shallowest-first (the -// BFS/highest-reuse-first ordering — a lexicographic scan would yield -// depth-first-within-subtree order, which under a budget pins a deep narrow -// slice), and pins from the front until budgetBytes is reached. -// -// Returns the number of branches pinned by this call (cumulative total via -// PinnedTotal). Calling LoadBulk again with a larger total budget extends the -// pinned set: it re-scans (a sequential range scan is cheap), skips the -// already-pinned shallowest entries, and pins the next budget's worth — the -// resumable/phased semantics of Run, implemented by re-scan. -func (p *ContractTrunkPreload) LoadBulk(budgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (newlyPinned int, err error) { - if cache == nil { - return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: cache is nil") - } - if rr == nil { - return 0, fmt.Errorf("ContractTrunkPreload.LoadBulk: range reader is nil") - } - if budgetBytes <= p.usedBytes { - return 0, nil - } - - evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(p.contractNibbles) - - var collected []bulkBranch - collect := func(from, to []byte) error { - return rr(from, to, func(key, value []byte, step uint64) bool { - path := nibbles.CompactToHex(key) - if len(path) < 64 || len(path) > maxStorageTrunkDepth || !bytes.Equal(path[:64], p.contractNibbles) { - return true // not a branch of this contract's subtree — skip, keep scanning - } - kc := make([]byte, len(key)) - copy(kc, key) - vc := make([]byte, len(value)) - copy(vc, value) - collected = append(collected, bulkBranch{key: kc, value: vc, step: step, depth: len(path)}) - return true - }) - } - if err = collect(evenFrom, evenTo); err != nil { - return 0, fmt.Errorf("LoadBulk even-range scan: %w", err) - } - if err = collect(oddFrom, oddTo); err != nil { - return 0, fmt.Errorf("LoadBulk odd-range scan: %w", err) - } - - // Shallowest first; tie-break by path bytes for determinism. - sort.Slice(collected, func(i, j int) bool { - if collected[i].depth != collected[j].depth { - return collected[i].depth < collected[j].depth - } - return bytes.Compare(collected[i].key, collected[j].key) < 0 - }) - - alreadyPinned := p.pinned // skip the entries a previous LoadBulk already pinned (same shallowest-first order) - chunkPinned := 0 - chunkBytes := 0 - for idx, b := range collected { - if idx < alreadyPinned { - continue - } - entryCost := estimatedEntryOverheadBytes + len(b.key) + len(b.value) - if p.usedBytes+chunkBytes+entryCost > budgetBytes { - break - } - cache.PinEntry(b.key, b.value, b.step, "preload-trunk-bulk") - p.pinnedPrefixes = append(p.pinnedPrefixes, b.key) - chunkBytes += entryCost - chunkPinned++ - if b.depth > p.maxDepthReached { - p.maxDepthReached = b.depth - } - if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { - logger.Info("[trunk-preload-bulk] progress", - "pinned", p.pinned+chunkPinned, "depth", b.depth, "used_mb", (p.usedBytes+chunkBytes)/(1<<20)) - } - } - p.pinned += chunkPinned - p.usedBytes += chunkBytes - p.lastBulkHadMore = p.pinned < len(collected) - if logger != nil { - logger.Info("[trunk-preload-bulk] scanned", "contract_hash", fmt.Sprintf("%x", p.contractHash), - "scanned", len(collected), "pinned_this_call", chunkPinned, "pinned_total", p.pinned, - "used_mb", p.usedBytes/(1<<20), "max_depth_reached", p.maxDepthReached, "budget_exhausted", p.pinned < len(collected)) - } - return chunkPinned, nil -} - -// PreloadContractTrunkBulk is the bulk analogue of PreloadContractTrunk: one -// call, bounded by ramBudgetBytes, using a range scan. -func PreloadContractTrunkBulk(contractHash []byte, ramBudgetBytes int, rr CommitmentRangeReader, cache *BranchCache, logger log.Logger) (int, error) { - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunkBulk: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - p, err := NewContractTrunkPreload(contractHash) - if err != nil { - return 0, err - } - return p.LoadBulk(ramBudgetBytes, rr, cache, logger) -} diff --git a/execution/commitment/preload_bulk_test.go b/execution/commitment/preload_bulk_test.go deleted file mode 100644 index 4b0ab650ac4..00000000000 --- a/execution/commitment/preload_bulk_test.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2026 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. - -package commitment - -import ( - "bytes" - "encoding/binary" - "sort" - "testing" - - "github.com/erigontech/erigon/execution/commitment/nibbles" -) - -func hexNibbles(b []byte) []byte { - out := make([]byte, len(b)*2) - for i, x := range b { - out[2*i] = x >> 4 - out[2*i+1] = x & 0x0f - } - return out -} - -// keyForBranch builds the CommitmentDomain key for a branch of contract H's -// storage subtree reached by the given slot-path nibbles. -func keyForBranch(contractNibbles, slotPath []byte) []byte { - full := make([]byte, 0, len(contractNibbles)+len(slotPath)) - full = append(full, contractNibbles...) - full = append(full, slotPath...) - return nibbles.HexToCompact(full) -} - -func inRange(k, from, to []byte) bool { - return bytes.Compare(k, from) >= 0 && (to == nil || bytes.Compare(k, to) < 0) -} - -func TestContractTrunkKeyRanges(t *testing.T) { - hashA := make([]byte, 32) - for i := range hashA { - hashA[i] = byte(7*i + 3) // arbitrary - } - hashB := make([]byte, 32) - for i := range hashB { - hashB[i] = byte(251 - 3*i) - } - nibA := hexNibbles(hashA) - nibB := hexNibbles(hashB) - - evenFrom, evenTo, oddFrom, oddTo := contractTrunkKeyRanges(nibA) - - // Every branch of A at assorted slot-paths must land in exactly one range, - // per the parity of its total nibble length. - slotPaths := [][]byte{ - {}, // depth 64 — subtree root - {0x0}, // 65 - {0xf}, // 65 - {0x1, 0x2}, // 66 - {0xf, 0xf}, // 66 - {0x3, 0x4, 0x5}, // 67 - {0x6, 0x7, 0x8, 0x9}, // 68 - {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 - make([]byte, 64), // 128 — deepest - } - for _, sp := range slotPaths { - k := keyForBranch(nibA, sp) - total := 64 + len(sp) - if total%2 == 0 { - if !inRange(k, evenFrom, evenTo) { - t.Fatalf("depth %d branch %x not in even range [%x,%x)", total, k, evenFrom, evenTo) - } - if inRange(k, oddFrom, oddTo) { - t.Fatalf("depth %d (even) branch %x unexpectedly in odd range", total, k) - } - } else { - if !inRange(k, oddFrom, oddTo) { - t.Fatalf("depth %d branch %x not in odd range [%x,%x)", total, k, oddFrom, oddTo) - } - if inRange(k, evenFrom, evenTo) { - t.Fatalf("depth %d (odd) branch %x unexpectedly in even range", total, k) - } - } - // round-trip: CompactToHex(key) must reproduce the full path - got := nibbles.CompactToHex(k) - want := append(append([]byte{}, nibA...), sp...) - if !bytes.Equal(got, want) { - t.Fatalf("CompactToHex round-trip mismatch: got %x want %x", got, want) - } - } - - // A different contract's branches must be in neither of A's ranges. - for _, sp := range slotPaths[:6] { - k := keyForBranch(nibB, sp) - if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { - t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) - } - } -} - -func TestNextSubtree(t *testing.T) { - cases := []struct{ in, want []byte }{ - {[]byte{0x01, 0x02}, []byte{0x01, 0x03}}, - {[]byte{0x01, 0xff}, []byte{0x02}}, - {[]byte{0x00}, []byte{0x01}}, - } - for _, c := range cases { - if got := nextSubtree(c.in); !bytes.Equal(got, c.want) { - t.Fatalf("nextSubtree(%x) = %x, want %x", c.in, got, c.want) - } - } - if nextSubtree([]byte{0xff, 0xff}) != nil { - t.Fatalf("nextSubtree(0xffff) should be nil") - } -} - -// branchVal returns a deterministic branch-node value of size sz with a valid -// 4-byte header (touchMap||afterMap) — afterMap=0 so the BFS-children logic -// (in Run, not LoadBulk) would queue nothing; LoadBulk ignores the bitmap. -func branchVal(seed, sz int) []byte { - if sz < 4 { - sz = 4 - } - v := make([]byte, sz) - binary.BigEndian.PutUint16(v[0:2], uint16(seed)) - for i := 4; i < sz; i++ { - v[i] = byte(seed + i) - } - return v -} - -func TestLoadBulk_ShallowestFirstWithinBudget(t *testing.T) { - hash := make([]byte, 32) - for i := range hash { - hash[i] = byte(0x40 + i) - } - nib := hexNibbles(hash) - - // Synthetic branch set at depths 64..69, mixed parity, fixed value size. - const valSz = 100 - type ent struct { - key []byte - val []byte - depth int - } - var ents []ent - add := func(sp []byte) { - ents = append(ents, ent{key: keyForBranch(nib, sp), val: branchVal(len(ents)+1, valSz), depth: 64 + len(sp)}) - } - add([]byte{}) // 64 - add([]byte{0x1}) // 65 - add([]byte{0x2}) // 65 - add([]byte{0x3, 0x4}) // 66 - add([]byte{0x5, 0x6}) // 66 - add([]byte{0x7, 0x8, 0x9}) // 67 - add([]byte{0xa, 0xb, 0xc, 0xd}) // 68 - add([]byte{0xe, 0xf, 0x0, 0x1, 0x2}) // 69 - - // Fake range reader: yields the entries whose key falls in [from,to), in - // ascending key order. - rr := CommitmentRangeReader(func(from, to []byte, yield func(k, v []byte, step uint64) bool) error { - var in []ent - for _, e := range ents { - if bytes.Compare(e.key, from) >= 0 && (to == nil || bytes.Compare(e.key, to) < 0) { - in = append(in, e) - } - } - sort.Slice(in, func(i, j int) bool { return bytes.Compare(in[i].key, in[j].key) < 0 }) - for _, e := range in { - if !yield(e.key, e.val, 1) { - return nil - } - } - return nil - }) - - entryCost := estimatedEntryOverheadBytes + 33 /*≈compact key len*/ + valSz // approx; key lens are 33-36 - // budget for ~the 4 shallowest (depths 64,65,65,66) — give a bit of slack - budget := 4*entryCost + 80 - - c := NewBranchCache(100) - p, err := NewContractTrunkPreload(hash) - if err != nil { - t.Fatal(err) - } - n, err := p.LoadBulk(budget, rr, c, nil) - if err != nil { - t.Fatal(err) - } - if n == 0 || n != p.PinnedTotal() { - t.Fatalf("LoadBulk pinned %d (PinnedTotal=%d), expected >0 and consistent", n, p.PinnedTotal()) - } - if c.PinnedCount() != n { - t.Fatalf("cache PinnedCount=%d != pinned %d", c.PinnedCount(), n) - } - // Sort the synthetic set the way LoadBulk does (shallowest first, key tie-break). - sortedExpected := append([]ent(nil), ents...) - sort.Slice(sortedExpected, func(i, j int) bool { - if sortedExpected[i].depth != sortedExpected[j].depth { - return sortedExpected[i].depth < sortedExpected[j].depth - } - return bytes.Compare(sortedExpected[i].key, sortedExpected[j].key) < 0 - }) - // The first n pinned must be exactly the n shallowest. - for i := 0; i < n; i++ { - v, _, ok := c.Get(sortedExpected[i].key) - if !ok { - t.Fatalf("expected branch #%d (depth %d, key %x) to be pinned, but it's not in cache", i, sortedExpected[i].depth, sortedExpected[i].key) - } - if !bytes.Equal(v, sortedExpected[i].val) { - t.Fatalf("pinned branch #%d value mismatch", i) - } - } - // The (n+1)-th shallowest must NOT be pinned (budget cut it off). - if n < len(ents) { - if _, _, ok := c.Get(sortedExpected[n].key); ok { - t.Fatalf("branch #%d should have been excluded by the budget but is pinned", n) - } - } - if p.MaxDepthReached() != sortedExpected[n-1].depth { - t.Fatalf("MaxDepthReached=%d, expected %d", p.MaxDepthReached(), sortedExpected[n-1].depth) - } - - // Phased extend: a larger total budget pins the rest. - n2, err := p.LoadBulk(1<<20, rr, c, nil) - if err != nil { - t.Fatal(err) - } - if n2 == 0 || p.PinnedTotal() != len(ents) { - t.Fatalf("extend: pinned %d more, total %d, expected total %d", n2, p.PinnedTotal(), len(ents)) - } - if c.PinnedCount() != len(ents) { - t.Fatalf("after extend, cache PinnedCount=%d != %d", c.PinnedCount(), len(ents)) - } -} diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index 588d86de649..edf8ebac6b0 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -189,7 +189,7 @@ func TestAdaptivePinController_PromoteOnThreshold(t *testing.T) { } // OnBlockComplete should promote the contract. - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1) require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) require.Greater(t, c.PinnedCount(), 0) @@ -215,17 +215,17 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { // Hot block: triggers promotion. _, _, _ = c.Get(prefix) - ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1) pinnedAfterPromote := c.PinnedCount() require.Greater(t, pinnedAfterPromote, 0) // Cold blocks: no misses for the contract. After cooldown the // controller demotes and invalidates the pin set. - ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 2, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 1, "still pinned after 1 cold block") - ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false), nil) + ctrl.OnBlockComplete(context.Background(), 3, fakeReader(false)) require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } From 3e0154f245b3329e0299b8a3ec64ac414484df6e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 15:57:51 +0000 Subject: [PATCH 065/120] perf(preload): parallel file-only wave-BFS preload (cut-down) Replaces the per-prefix serial BFS-via-GetLatest preload (which crossed the MDBX boundary ~89k times per contract and finished in ~31s, losing the preload-vs-test-block race in the no-warmup harness) with a parallel, breadth-first, file-layer-only wave walk: PreloadContractTrunkParallel walks the contract subtree one depth level (wave) at a time; each wave's branches are resolved via a single batched resolver (BatchBranchResolver) that the caller fans out across worker RoTxs in contiguous, key-sorted slices -> locally sequential reads that drive page-cache readahead. Reads go via DebugGetLatestFromFiles (file layer only: no MDBX cursor, no per-key cgo crossing; values already dereferenced via replaceShortenedKeysInBranch). Breadth-first so a budget-truncated run still pins the shallowest, highest-reuse branches. triggerTrunkPreload wires it (PIN_TRUNK_PARALLEL, default on; =false falls back to the per-prefix BFS): N=GOMAXPROCS worker RoTxs (each goroutine its own tx -- MDBX cursors aren't concurrent-safe under one tx), contiguous-slice fanout. File-layer-only is correct for the from-cold-snapshot trigger path (MDBX holds nothing for the subtree). The MDBX-resident set (steady-state) is handled by a separate range-cursor prefetch -- see agentspecs/commitment-branch-preload-redesign.md; not in this cut-down. Unit tests: PreloadParallel_{FullBudget_BreadthFirst, BudgetCutoff, NotFoundStopsDescent, ResolverError}. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 107 +++++++- execution/commitment/preload_parallel.go | 166 ++++++++++++ execution/commitment/preload_parallel_test.go | 250 ++++++++++++++++++ 3 files changed, 510 insertions(+), 13 deletions(-) create mode 100644 execution/commitment/preload_parallel.go create mode 100644 execution/commitment/preload_parallel_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index da2b9a97610..8f3e1da3860 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1350,28 +1350,109 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach return } go func() { - tx, err := db.BeginTemporalRo(ctx) - if err != nil { - logger.Warn("[trunk-preload] BeginTemporalRo failed", "err", err) - return + // PIN_TRUNK_PARALLEL (default on): the parallel file-only wave-BFS + // (PreloadContractTrunkParallel). =false falls back to the per-prefix + // BFS via tx.GetLatest. Worker txs: each goroutine MUST use its own + // tx — MDBX cursors aren't concurrent-safe under one tx (cgo "unpinned + // Go pointer" panic). Reads here go via the file layer only + // (DebugGetLatestFromFiles): no MDBX cursor, no per-key cgo crossing, + // and values are already dereferenced. Correct for the from-cold- + // snapshot trigger path (MDBX holds nothing for the subtree). + parallel := dbg.EnvBool("PIN_TRUNK_PARALLEL", true) + nWorkers := runtime.GOMAXPROCS(0) + if nWorkers < 1 { + nWorkers = 1 + } + if nWorkers > 16 { + nWorkers = 16 + } + if !parallel { + nWorkers = 1 } - defer tx.Rollback() - // Read directly via tx.GetLatest — no SharedDomains wrap. - // SD's GetLatest layers in sd.mem / parent.mem checks that - // (a) we don't need (the trunk hasn't been written yet in this - // goroutine's tx), and (b) would risk re-introducing the - // shared-cursor problem. + txs := make([]kv.TemporalTx, 0, nWorkers) + for i := 0; i < nWorkers; i++ { + t, err := db.BeginTemporalRo(ctx) //nolint:gocritic // collected into txs; all rolled back via the defer below + if err != nil { + for _, t := range txs { + t.Rollback() + } + logger.Warn("[trunk-preload] BeginTemporalRo failed", "worker", i, "err", err) + return + } + txs = append(txs, t) + } + defer func() { + for _, t := range txs { + t.Rollback() + } + }() + reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := tx.GetLatest(kv.CommitmentDomain, prefix) + v, step, err := txs[0].GetLatest(kv.CommitmentDomain, prefix) if err != nil { return nil, 0, false, err } return v, uint64(step), len(v) > 0, nil } + + resolve := func(keys [][]byte) ([][]byte, error) { + n := len(keys) + vals := make([][]byte, n) + if n == 0 { + return vals, nil + } + nw := nWorkers + if nw > n { + nw = n + } + chunk := (n + nw - 1) / nw + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + for w := 0; w < nw; w++ { + lo, hi := w*chunk, (w+1)*chunk + if hi > n { + hi = n + } + if lo >= hi { + break + } + wg.Add(1) + go func(t kv.TemporalTx, lo, hi int) { + defer wg.Done() + d := t.Debug() + for i := lo; i < hi; i++ { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, keys[i], 0) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + return + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } + } + }(txs[w], lo, hi) + } + wg.Wait() + return vals, firstErr + } + for i, h := range hashes { started := time.Now() - logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h)) - n, err := commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "parallel", parallel, "workers", nWorkers) + var n int + var err error + if parallel { + n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, resolve, branchCache, logger) + } else { + n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) + } took := time.Since(started) if err != nil { logger.Warn("[trunk-preload] failed", diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go new file mode 100644 index 00000000000..1a88c6fd304 --- /dev/null +++ b/execution/commitment/preload_parallel.go @@ -0,0 +1,166 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// BatchBranchResolver resolves a batch of compact-encoded CommitmentDomain +// keys to their (already-dereferenced) branch-node values, reading from the +// file layer only — no MDBX cursor, hence no per-key cgo boundary crossing. +// The implementation is expected to fan the batch out across worker RoTxs; +// keys are passed in ascending key order so a contiguous-slice partition maps +// to contiguous file regions (drives page-cache readahead). Results MUST be +// returned in the same order as keys, with vals[i] == nil for keys not present +// in the file layer. +// +// Decoupled (callback form) so the commitment package needn't import db/state. +type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) + +// PreloadContractTrunkParallel pins contract H's storage-trunk branches into +// cache by walking the subtree breadth-first, one depth level ("wave") at a +// time, resolving each wave's branches through a single batched, parallelisable +// file-only read. Breadth-first so a budget-truncated run still pins the +// shallowest (highest-reuse) branches first; wave-batched so the resolver can +// sort+partition the reads for locality instead of issuing them one at a time. +// +// Bounded by ramBudgetBytes (same accounting as the BFS ContractTrunkPreload). +// Returns the number of branches pinned. +// +// File-layer-only: a branch present only in MDBX (recently written, not yet in +// a .kv file) is invisible here, and a branch present in both MDBX (fresh) and +// files (stale) resolves to the stale value. That is correct for the +// from-cold-snapshot trigger path (MDBX holds nothing for the subtree) — the +// case PIN_CONTRACT_TRUNKS targets. The MDBX-resident set is handled separately +// (a single range-cursor prefetch); see agentspecs/commitment-branch-preload-redesign.md. +func PreloadContractTrunkParallel( + contractHash []byte, + ramBudgetBytes int, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (pinned int, err error) { + if cache == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: cache is nil") + } + if resolve == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + } + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + if len(contractHash) != 32 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) + } + + contractNibbles := make([]byte, 64) + for i, b := range contractHash { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + type pathKey struct { + path []byte // nibble path (1 byte / nibble) + key []byte // HexToCompact(path) — the CommitmentDomain key + } + toPathKey := func(path []byte) pathKey { + k := nibbles.HexToCompact(path) + // HexToCompact may return a slice over a reused buffer. + kc := make([]byte, len(k)) + copy(kc, k) + return pathKey{path: path, key: kc} + } + + frontier := []pathKey{toPathKey(contractNibbles)} + usedBytes := 0 + maxDepthReached := 64 + budgetHit := false + + for depth := 64; depth <= maxStorageTrunkDepth && len(frontier) > 0 && !budgetHit; depth++ { + // Ascending key order so the resolver's contiguous partition is + // contiguous-in-file. BFS append order is already key-sorted (the + // HexToCompact packing is monotone for same-length paths), but sort + // defensively — it's a few k entries / wave, cheap. + sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + + keys := make([][]byte, len(frontier)) + for i := range frontier { + keys[i] = frontier[i].key + } + vals, rerr := resolve(keys) + if rerr != nil { + return pinned, fmt.Errorf("preload at depth %d: %w", depth, rerr) + } + if len(vals) != len(keys) { + return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(vals), len(keys)) + } + + var next []pathKey + for i, pk := range frontier { + v := vals[i] + if v == nil { + continue // not in the file layer + } + entryCost := estimatedEntryOverheadBytes + len(pk.key) + len(v) + if usedBytes+entryCost > ramBudgetBytes { + budgetHit = true + break + } + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") + usedBytes += entryCost + pinned++ + if depth > maxDepthReached { + maxDepthReached = depth + } + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload-parallel] progress", + "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + } + // Decode the branch header (2-byte touchMap || 2-byte afterMap || + // per-child data) and queue the children that exist in the trie. + if len(v) < 4 { + continue + } + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1<> 4 + out[2*i+1] = x & 0x0f + } + return out +} + +// branchVal builds a synthetic branch-node value: 2-byte touchMap (0) || +// 2-byte afterMap (the child bitmap) || zero-padding to size sz (>= 4). +func branchVal(afterMap uint16, sz int) []byte { + if sz < 4 { + sz = 4 + } + v := make([]byte, sz) + binary.BigEndian.PutUint16(v[2:4], afterMap) + return v +} + +// syntheticTree describes a contract storage subtree: path-string -> afterMap. +// Root is the 64-nibble path of the contract hash; a node R is present iff R is +// a key here, and R's children are R||n for each set bit n in afterMap[R]. +type syntheticTree map[string]uint16 + +func buildSyntheticTree(t *testing.T) (hash []byte, tree syntheticTree, allPaths [][]byte) { + t.Helper() + hash = make([]byte, 32) + for i := range hash { + hash[i] = 0x42 + } + root := string(hexNibbles(hash)) + // R(64) -> {1,2} ; R1(65) -> {3} ; R2(65) -> {4,5} ; + // R1.3(66) leaf ; R2.4(66) leaf ; R2.5(66) -> {6} ; R2.5.6(67) leaf. + r := []byte(root) + p := func(suffix ...byte) []byte { return append(append([]byte{}, r...), suffix...) } + tree = syntheticTree{ + string(p()): 0b110, // bits 1,2 + string(p(1)): 0b1000, // bit 3 + string(p(2)): 0b110000, // bits 4,5 + string(p(1, 3)): 0, + string(p(2, 4)): 0, + string(p(2, 5)): 0b1000000, // bit 6 + string(p(2, 5, 6)): 0, + } + for k := range tree { + allPaths = append(allPaths, []byte(k)) + } + return hash, tree, allPaths +} + +// fakeResolver returns a BatchBranchResolver backed by the synthetic tree. +// notFound (path-strings) are treated as absent from the file layer. +// valSz is the branch value size. If failOnKey is non-empty, the resolver +// returns an error when that key is requested. +func fakeResolver(tree syntheticTree, notFound map[string]bool, valSz int, failOnPath string) BatchBranchResolver { + return func(keys [][]byte) ([][]byte, error) { + // keys must be sorted ascending (the contract of BatchBranchResolver). + for i := 1; i < len(keys); i++ { + if bytes.Compare(keys[i-1], keys[i]) >= 0 { + return nil, errors.New("resolver got unsorted keys") + } + } + vals := make([][]byte, len(keys)) + for i, k := range keys { + path := string(nibbles.CompactToHex(k)) + if failOnPath != "" && path == failOnPath { + return nil, errors.New("synthetic resolver failure") + } + am, ok := tree[path] + if !ok || notFound[path] { + continue // nil + } + vals[i] = branchVal(am, valSz) + } + return vals, nil + } +} + +// breadthFirstOrder returns the synthetic tree's paths in the order +// PreloadContractTrunkParallel pins them: by depth, then by compact-key. +func breadthFirstOrder(tree syntheticTree, exclude map[string]bool) []string { + type pk struct { + path string + key []byte + } + var pks []pk + // reachability from root, honoring exclude (an excluded node stops descent — + // it and its subtree become unreachable) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + reach := map[string]bool{} + var dfs func(p string) + dfs = func(p string) { + if exclude[p] { + return + } + am, ok := tree[p] + if !ok { + return + } + reach[p] = true + for n := 0; n < 16; n++ { + if am&(1< root pinned at depth 64, then error. + _, err := PreloadContractTrunkParallel(hash, 1<<20, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) + if err == nil { + t.Fatal("expected error from the resolver") + } + if c.PinnedCount() == 0 { + t.Fatal("the depth-64 root should have been pinned before the depth-65 failure") + } +} From 4c9968a4370c86d75b3149e1d4adeba0a7904c3b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 22:19:33 +0000 Subject: [PATCH 066/120] perf(preload): cap per-wave fetch by remaining budget PreloadContractTrunkParallel resolved an entire depth wave before applying the RAM budget. Depth 69 of the SSTORE-bloat trunk is ~1.3M keys but only ~20k fit the remaining budget -- so it issued >1M file lookups to pin ~20k, and that over-fetch ran concurrently with the test block (docker bench: 10.1 mgas/s, preload 38.5s vs the BFS baseline's 13.2 / 31s -- depths 64-68 pinned in ~1-2s, then 31s stalled on the over-fetched depth-69 wave). Now: each wave's fetch is capped at remaining_budget/minEntryBytes + 1, where minEntryBytes (= overhead + shortest depth>=64 compact key) is a true lower bound on a pinned entry's cost -- so it never under-fetches and the budget is guaranteed exhausted within the (truncated) wave; if a wave is truncated we stop after it. Depth 69 goes from ~1.3M lookups -> ~tens of thousands -> ~seconds, the preload finishes well before the block, and the over-fetch contention is gone. Adds TestPreloadParallel_CapsWaveFetch. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload_parallel.go | 18 +++++++++ execution/commitment/preload_parallel_test.go | 39 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 1a88c6fd304..057a22fdfcd 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -96,6 +96,21 @@ func PreloadContractTrunkParallel( // defensively — it's a few k entries / wave, cheap. sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + // Cap the wave's *fetch* at what the remaining budget can possibly + // absorb (minEntryBytes is a true lower bound on a pinned entry's cost, + // so this never under-fetches and the budget is guaranteed exhausted + // inside the wave). Without it, a wide budget-truncated wave — depth 69 + // is ~1.3M keys but only ~20k fit — would issue >1M file lookups to pin + // a tiny fraction, and that over-fetch races the next block. + const minEntryBytes = estimatedEntryOverheadBytes + 33 // 33 = shortest compact key at depth >= 64; value >= 0 bytes + truncatedByBudget := false + if remaining := ramBudgetBytes - usedBytes; remaining > 0 { + if maxFetch := remaining/minEntryBytes + 1; maxFetch < len(frontier) { + frontier = frontier[:maxFetch] + truncatedByBudget = true + } + } + keys := make([][]byte, len(frontier)) for i := range frontier { keys[i] = frontier[i].key @@ -145,6 +160,9 @@ func PreloadContractTrunkParallel( next = append(next, toPathKey(childPath)) } } + if truncatedByBudget { + budgetHit = true // belt-and-braces: the wave was sized to exhaust the budget + } frontier = next } diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go index 0d804f616ea..11a44acd445 100644 --- a/execution/commitment/preload_parallel_test.go +++ b/execution/commitment/preload_parallel_test.go @@ -230,6 +230,45 @@ func TestPreloadParallel_NotFoundStopsDescent(t *testing.T) { } } +func TestPreloadParallel_CapsWaveFetch(t *testing.T) { + // A wide wave (root with all 16 children) under a tiny budget: the depth-65 + // fetch must be capped well below the wave width — otherwise a budget- + // truncated wide wave (depth 69 on the real workload is ~1.3M keys) would + // fetch the whole thing to pin a handful. + hash := make([]byte, 32) + for i := range hash { + hash[i] = 0x55 + } + root := string(hexNibbles(hash)) + tree := syntheticTree{root: 0xffff} + for n := 0; n < 16; n++ { + tree[root+string([]byte{byte(n)})] = 0 // 16 depth-65 leaves + } + const valSz = 100 + entry := estimatedEntryOverheadBytes + len(nibbles.HexToCompact([]byte(root))) + valSz + budget := 3*entry + 50 // ~3 entries + + maxBatch := 0 + base := fakeResolver(tree, nil, valSz, "") + resolve := func(keys [][]byte) ([][]byte, error) { + if len(keys) > maxBatch { + maxBatch = len(keys) + } + return base(keys) + } + c := NewBranchCache(64) + n, err := PreloadContractTrunkParallel(hash, budget, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if n < 1 || n > 3 { + t.Fatalf("pinned %d, expected 1..3 for a ~3-entry budget", n) + } + if maxBatch > 6 { + t.Fatalf("depth-65 wave (width 16) should have been capped to ~remaining/minEntryBytes; resolver saw a batch of %d", maxBatch) + } +} + func TestPreloadParallel_ResolverError(t *testing.T) { hash, tree, _ := buildSyntheticTree(t) root := "" From eb0c0c5803d0c388bfcd46aa25f5f7a622073fff Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 12 May 2026 22:47:40 +0000 Subject: [PATCH 067/120] =?UTF-8?q?perf(preload):=20phase=201=20=E2=80=94?= =?UTF-8?q?=20MDBX-resident=20branch=20prefetch=20(operational=20correctne?= =?UTF-8?q?ss)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parallel wave-BFS preload was file-layer-only: a branch present in MDBX but not yet flushed to a .kv file is invisible, and one present in both (DB fresh, file stale) resolved to the stale value. Correct from a cold snapshot (MDBX empty for the subtree), wrong on a live node. Phase 1: triggerTrunkPreload now first scans the contract's two CommitmentDomain key ranges (ContractTrunkKeyRanges, re-added with NextSubtree from the deleted preload_bulk.go) over TblCommitmentVals via one DupSort cursor per range (dups are invertedStep||value, latest-first, so Seek + NextNoDup walks the latest value per key) into an in-memory dbBranches map, bounded by the RAM budget. PreloadContractTrunkParallel now takes that map: per wave, DB-hits are resolved from memory (the freshest values, pinned first), file-misses go through the batched file resolver (capped by the budget remaining after the DB-hits). On a from-cold-snapshot start dbBranches is empty -> behaviour unchanged. Tests: TestPreloadParallel_DbHitsShadowFiles (DB value wins over a stale file value and its bitmap drives descent), TestNextSubtree, TestContractTrunkKeyRanges. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 53 ++++- execution/commitment/preload_parallel.go | 197 +++++++++++------- execution/commitment/preload_parallel_test.go | 122 ++++++++++- execution/commitment/preload_ranges.go | 62 ++++++ 4 files changed, 351 insertions(+), 83 deletions(-) create mode 100644 execution/commitment/preload_ranges.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8f3e1da3860..986a7519492 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1443,13 +1443,64 @@ func triggerTrunkPreload(ctx context.Context, branchCache *commitment.BranchCach return vals, firstErr } + // dbPrefetch: phase 1 — pull the contract's MDBX-resident commitment + // branches (the freshest values: recently rewritten, not yet flushed to + // files) into an in-memory map via one range cursor per parity range, so + // the parallel wave-BFS resolves them from memory and the file layer is + // authoritative only for the keys MDBX doesn't have. The DupSort + // valsTable dups are invertedStep||value (latest first), so Seek + + // NextNoDup walks the latest value per key. Bounded by ramBudgetBytes; + // on a from-cold-snapshot start this finds nothing. + dbPrefetch := func(h []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := txs[0].CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + logger.Warn("[trunk-preload] phase-1 CursorDupSort failed", "err", cerr) + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(h)) + bytesBudget := ramBudgetBytes + scanRange := func(from, to []byte) error { + for k, v, err := c.Seek(from); k != nil; k, v, err = c.NextNoDup() { + if err != nil { + return err + } + if bytes.Compare(k, to) >= 0 { + break + } + if len(v) < 8 { + continue + } + val := common.Copy(v[8:]) + m[string(common.Copy(k))] = val + if bytesBudget -= len(val) + 8; bytesBudget <= 0 { + return nil + } + } + return nil + } + if err := scanRange(evenFrom, evenTo); err != nil { + logger.Warn("[trunk-preload] phase-1 even-range scan", "err", err) + } else if bytesBudget > 0 { + if err := scanRange(oddFrom, oddTo); err != nil { + logger.Warn("[trunk-preload] phase-1 odd-range scan", "err", err) + } + } + return m + } + for i, h := range hashes { started := time.Now() logger.Info("[trunk-preload] contract starting", "i", i, "hash", hex.EncodeToString(h), "parallel", parallel, "workers", nWorkers) var n int var err error if parallel { - n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, resolve, branchCache, logger) + dbBranches := dbPrefetch(h) + if len(dbBranches) > 0 { + logger.Info("[trunk-preload] phase-1 db prefetch", "hash", hex.EncodeToString(h), "db_branches", len(dbBranches)) + } + n, err = commitment.PreloadContractTrunkParallel(h, ramBudgetBytes, dbBranches, resolve, branchCache, logger) } else { n, err = commitment.PreloadContractTrunk(h, ramBudgetBytes, reader, branchCache, logger) } diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 057a22fdfcd..3231c4a5e66 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -30,25 +30,40 @@ import ( // Decoupled (callback form) so the commitment package needn't import db/state. type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) +// estimatedEntryCost is the predicted RAM cost of pinning a (key, value) branch +// entry — see estimatedEntryOverheadBytes in preload.go. +func estimatedEntryCost(key, value []byte) int { + return estimatedEntryOverheadBytes + len(key) + len(value) +} + +// minEntryBytes is a true lower bound on estimatedEntryCost for a storage-trunk +// branch (33 = shortest HexToCompact key at depth >= 64; value may be empty). +// Used to bound a wave's *file fetch*: capping at remaining/minEntryBytes+1 +// never under-fetches, so the budget is guaranteed exhausted inside the wave. +const minEntryBytes = estimatedEntryOverheadBytes + 33 + // PreloadContractTrunkParallel pins contract H's storage-trunk branches into // cache by walking the subtree breadth-first, one depth level ("wave") at a -// time, resolving each wave's branches through a single batched, parallelisable -// file-only read. Breadth-first so a budget-truncated run still pins the -// shallowest (highest-reuse) branches first; wave-batched so the resolver can -// sort+partition the reads for locality instead of issuing them one at a time. +// time. Within a wave, each branch is resolved either from dbBranches (the +// MDBX-resident set — the freshest values; supplied by the caller's phase-1 +// range-cursor prefetch, may be nil/empty) or, for keys absent from it, through +// a single batched, parallelisable file-only read (BatchBranchResolver). // -// Bounded by ramBudgetBytes (same accounting as the BFS ContractTrunkPreload). -// Returns the number of branches pinned. +// Breadth-first so a budget-truncated run still pins the shallowest, highest- +// reuse branches first; wave-batched so the file resolver can sort+partition the +// reads for locality. DB-hits are pinned before file-hits within a wave (the DB +// holds the freshest, most-reused branches of a hot contract). Bounded by +// ramBudgetBytes. Returns the number of branches pinned. // -// File-layer-only: a branch present only in MDBX (recently written, not yet in -// a .kv file) is invisible here, and a branch present in both MDBX (fresh) and -// files (stale) resolves to the stale value. That is correct for the -// from-cold-snapshot trigger path (MDBX holds nothing for the subtree) — the -// case PIN_CONTRACT_TRUNKS targets. The MDBX-resident set is handled separately -// (a single range-cursor prefetch); see agentspecs/commitment-branch-preload-redesign.md. +// Correctness: dbBranches values shadow file values for the same key (DB is +// authoritative for steps not yet flushed to files), so passing the DB-resident +// set is what makes this safe on a live node. With an empty dbBranches it's a +// pure file-only preload — correct only from a cold snapshot, where MDBX holds +// nothing for the subtree. func PreloadContractTrunkParallel( contractHash []byte, ramBudgetBytes int, + dbBranches map[string][]byte, resolve BatchBranchResolver, cache *BranchCache, logger log.Logger, @@ -66,102 +81,129 @@ func PreloadContractTrunkParallel( return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) } - contractNibbles := make([]byte, 64) - for i, b := range contractHash { - contractNibbles[2*i] = b >> 4 - contractNibbles[2*i+1] = b & 0x0f - } - type pathKey struct { path []byte // nibble path (1 byte / nibble) key []byte // HexToCompact(path) — the CommitmentDomain key } toPathKey := func(path []byte) pathKey { k := nibbles.HexToCompact(path) - // HexToCompact may return a slice over a reused buffer. kc := make([]byte, len(k)) - copy(kc, k) + copy(kc, k) // HexToCompact result may alias a reused buffer return pathKey{path: path, key: kc} } - frontier := []pathKey{toPathKey(contractNibbles)} + frontier := []pathKey{toPathKey(ContractNibbles(contractHash))} usedBytes := 0 maxDepthReached := 64 budgetHit := false + var dbHitsPinned int + + // pin records the entry, advances accounting, and queues the branch's + // children. Returns false if the entry didn't fit (budget exhausted). + pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { + cost := estimatedEntryCost(pk.key, v) + if usedBytes+cost > ramBudgetBytes { + budgetHit = true + return false + } + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") + usedBytes += cost + pinned++ + if depth > maxDepthReached { + maxDepthReached = depth + } + if logger != nil && pinned%5000 == 0 { + logger.Info("[trunk-preload-parallel] progress", + "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + } + if len(v) >= 4 { // 2-byte touchMap || 2-byte afterMap || per-child data + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1< 0 && !budgetHit; depth++ { - // Ascending key order so the resolver's contiguous partition is - // contiguous-in-file. BFS append order is already key-sorted (the - // HexToCompact packing is monotone for same-length paths), but sort - // defensively — it's a few k entries / wave, cheap. + // Ascending key order so a contiguous-slice partition of the file batch + // is contiguous-in-file (page-cache readahead). BFS append order is + // already key-sorted (HexToCompact packing is monotone for same-length + // paths), but sort defensively — a few k entries / wave, cheap. sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) - // Cap the wave's *fetch* at what the remaining budget can possibly - // absorb (minEntryBytes is a true lower bound on a pinned entry's cost, - // so this never under-fetches and the budget is guaranteed exhausted - // inside the wave). Without it, a wide budget-truncated wave — depth 69 - // is ~1.3M keys but only ~20k fit — would issue >1M file lookups to pin - // a tiny fraction, and that over-fetch races the next block. - const minEntryBytes = estimatedEntryOverheadBytes + 33 // 33 = shortest compact key at depth >= 64; value >= 0 bytes + // Split this wave into DB-resident hits (have the value) and file-misses + // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). + var dbHits []pathKey + var dbVals [][]byte + var fileMiss []pathKey + dbHitsBytes := 0 + for _, pk := range frontier { + if v, ok := dbBranches[string(pk.key)]; ok { + dbHits = append(dbHits, pk) + dbVals = append(dbVals, v) + dbHitsBytes += estimatedEntryCost(pk.key, v) + } else { + fileMiss = append(fileMiss, pk) + } + } + + // Cap the *file fetch* at what the budget can absorb after the DB-hits + // (which we pin first). minEntryBytes is a true lower bound, so this + // never under-fetches; a truncated wave is guaranteed to exhaust the + // budget, so we stop after it. truncatedByBudget := false - if remaining := ramBudgetBytes - usedBytes; remaining > 0 { - if maxFetch := remaining/minEntryBytes + 1; maxFetch < len(frontier) { - frontier = frontier[:maxFetch] + if fileBudget := ramBudgetBytes - usedBytes - dbHitsBytes; fileBudget <= 0 { + if len(fileMiss) > 0 { + fileMiss = fileMiss[:0] truncatedByBudget = true } + } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { + fileMiss = fileMiss[:maxFileFetch] + truncatedByBudget = true } - keys := make([][]byte, len(frontier)) - for i := range frontier { - keys[i] = frontier[i].key - } - vals, rerr := resolve(keys) - if rerr != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", depth, rerr) - } - if len(vals) != len(keys) { - return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(vals), len(keys)) + var fileVals [][]byte + if len(fileMiss) > 0 { + keys := make([][]byte, len(fileMiss)) + for i := range fileMiss { + keys[i] = fileMiss[i].key + } + fileVals, err = resolve(keys) + if err != nil { + return pinned, fmt.Errorf("preload at depth %d: %w", depth, err) + } + if len(fileVals) != len(keys) { + return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) + } } var next []pathKey - for i, pk := range frontier { - v := vals[i] - if v == nil { - continue // not in the file layer - } - entryCost := estimatedEntryOverheadBytes + len(pk.key) + len(v) - if usedBytes+entryCost > ramBudgetBytes { - budgetHit = true + for i, pk := range dbHits { + if !pin(pk, dbVals[i], depth, &next) { break } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") - usedBytes += entryCost - pinned++ - if depth > maxDepthReached { - maxDepthReached = depth - } - if logger != nil && pinned%5000 == 0 { - logger.Info("[trunk-preload-parallel] progress", - "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) - } - // Decode the branch header (2-byte touchMap || 2-byte afterMap || - // per-child data) and queue the children that exist in the trie. - if len(v) < 4 { - continue - } - bitmap := binary.BigEndian.Uint16(v[2:4]) - for n := 0; n < 16; n++ { - if bitmap&(1<= 0 && (to == nil || bytes.Compare(k, to) < 0) + } + keyOf := func(contractNibbles, slotPath []byte) []byte { + return nibbles.HexToCompact(append(append([]byte{}, contractNibbles...), slotPath...)) + } + slotPaths := [][]byte{ + {}, // 64 — subtree root (even) + {0x0}, {0xf}, // 65 (odd) + {0x1, 0x2}, {0xf, 0xf}, // 66 (even) + {0x3, 0x4, 0x5}, // 67 (odd) + {0x6, 0x7, 0x8, 0x9}, // 68 (even) + {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 (odd) + make([]byte, 64), // 128 (even) — deepest + } + for _, sp := range slotPaths { + k := keyOf(nibA, sp) + total := 64 + len(sp) + if total%2 == 0 { + if !inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d (even) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, evenFrom, evenTo, oddFrom, oddTo) + } + } else { + if !inRange(k, oddFrom, oddTo) || inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d (odd) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, oddFrom, oddTo, evenFrom, evenTo) + } + } + if got := nibbles.CompactToHex(k); !bytes.Equal(got, append(append([]byte{}, nibA...), sp...)) { + t.Fatalf("CompactToHex round-trip mismatch for slot %x", sp) + } + } + // A different contract's branches must be in neither of A's ranges. + for _, sp := range slotPaths[:6] { + k := keyOf(nibB, sp) + if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) + } + } +} + func TestPreloadParallel_ResolverError(t *testing.T) { hash, tree, _ := buildSyntheticTree(t) root := "" @@ -279,7 +391,7 @@ func TestPreloadParallel_ResolverError(t *testing.T) { } c := NewBranchCache(64) // Fail when R||1 (depth 65) is requested -> root pinned at depth 64, then error. - _, err := PreloadContractTrunkParallel(hash, 1<<20, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) + _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) if err == nil { t.Fatal("expected error from the resolver") } diff --git a/execution/commitment/preload_ranges.go b/execution/commitment/preload_ranges.go new file mode 100644 index 00000000000..7883ad95681 --- /dev/null +++ b/execution/commitment/preload_ranges.go @@ -0,0 +1,62 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import "github.com/erigontech/erigon/execution/commitment/nibbles" + +// NextSubtree returns the smallest key K' that is greater than every key having +// `in` as a prefix — i.e. the exclusive upper bound for a prefix-range scan over +// `in`. Returns nil if `in` is all 0xff (no such K'). Mirrors db/kv.NextSubtree; +// inlined to keep this package's import set minimal. +func NextSubtree(in []byte) []byte { + r := make([]byte, len(in)) + copy(r, in) + for i := len(r) - 1; i >= 0; i-- { + if r[i] != 0xff { + r[i]++ + return r[:i+1] + } + } + return nil +} + +// ContractTrunkKeyRanges returns the two CommitmentDomain key ranges that +// together cover every branch node of a contract's storage subtree. +// +// The commitment domain keys branches by HexToCompact(nibblePath); HexToCompact's +// flag byte differs by the parity of the path length, so a contract's storage +// branches (path = keccak256(addr)'s 64 nibbles ++ k slot-path nibbles, total +// 64+k) split into two non-adjacent byte ranges: +// - even total length (k even, incl. the depth-64 subtree root): +// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) +// - odd total length (k odd): +// all such keys lie in [HexToCompact(H||0), NextSubtree(HexToCompact(H||15))). +func ContractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { + evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes + evenTo = NextSubtree(evenFrom) + + odd0 := make([]byte, 0, len(contractNibbles)+1) + odd0 = append(append(odd0, contractNibbles...), 0) + oddF := make([]byte, 0, len(contractNibbles)+1) + oddF = append(append(oddF, contractNibbles...), 15) + oddFrom = nibbles.HexToCompact(odd0) + oddTo = NextSubtree(nibbles.HexToCompact(oddF)) + return evenFrom, evenTo, oddFrom, oddTo +} + +// ContractNibbles expands a 32-byte contract hash to its 64-nibble path +// (one nibble per byte, high nibble first). +func ContractNibbles(contractHash []byte) []byte { + out := make([]byte, len(contractHash)*2) + for i, b := range contractHash { + out[2*i] = b >> 4 + out[2*i+1] = b & 0x0f + } + return out +} From a3374fed486c19d9e26316483021a46f0f133e68 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 08:18:15 +0000 Subject: [PATCH 068/120] db/state: test for the phase-1 dbPrefetch cursor walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestCommitmentPreloadPhase1Cursor populates TblCommitmentVals with real commitment-domain branch entries at multiple steps for two contracts, then walks the contract's two parity ranges (commitment.ContractTrunkKeyRanges) via CursorDupSort + Seek + NextNoDup and decodes v[8:] as the value — the same mechanics as triggerTrunkPreload's dbPrefetch. Asserts: - every contract-A branch appears in the cursor walk with its LATEST value (the dup decoded by skipping the 8-byte invertedStep prefix is correct); - the foreign contract's keys do not leak into contract-A's parity ranges (ContractTrunkKeyRanges is tight). This covers the cursor mechanics that the preload-side fake-resolver unit test (commitment.TestPreloadParallel_DbHitsShadowFiles) does not. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/commitment_preload_test.go | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 db/state/commitment_preload_test.go diff --git a/db/state/commitment_preload_test.go b/db/state/commitment_preload_test.go new file mode 100644 index 00000000000..c7eb557f4a7 --- /dev/null +++ b/db/state/commitment_preload_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package state + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/state/statecfg" + "github.com/erigontech/erigon/execution/commitment" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// TestCommitmentPreloadPhase1Cursor mirrors triggerTrunkPreload's phase-1 +// dbPrefetch in db/state/execctx/domain_shared.go: a DupSort cursor over +// TblCommitmentVals walked via Seek + NextNoDup, decoding v[8:] as the value +// (dups = invertedStep||value, latest-first). Asserts that for a populated +// CommitmentDomain DB, the walk over the contract's two parity ranges +// (commitment.ContractTrunkKeyRanges) picks up the LATEST value per key and +// excludes keys outside the ranges (foreign contract). +// +// The preload-side unit test (commitment.TestPreloadParallel_DbHitsShadowFiles) +// covers the algorithm with a fake dbBranches map; this covers the cursor +// mechanics against the real domain writer encoding. +func TestCommitmentPreloadPhase1Cursor(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + logger := log.New() + db, d := testDbAndDomainOfStep(t, statecfg.Schema.CommitmentDomain, 16, logger) + + ctx := t.Context() + tx, err := db.BeginRw(ctx) + require.NoError(t, err) + defer tx.Rollback() + + domainRoTx := d.beginForTests() + defer domainRoTx.Close() + writer := domainRoTx.NewWriter() + defer writer.Close() + + // Pick two contracts so we can also assert foreign-contract keys are + // excluded from the target contract's parity ranges. + contractA := make([]byte, 32) + for i := range contractA { + contractA[i] = byte(0x10 + i) + } + contractB := make([]byte, 32) + for i := range contractB { + contractB[i] = byte(0x80 - i) + } + nibA := commitment.ContractNibbles(contractA) + nibB := commitment.ContractNibbles(contractB) + + // keyAt builds a real commitment-domain key: HexToCompact(contractNibbles + // ++ slotPath). The path's parity falls out of len(slotPath). + keyAt := func(contractNibbles, slotPath []byte) []byte { + full := append(append([]byte{}, contractNibbles...), slotPath...) + k := nibbles.HexToCompact(full) + return common.Copy(k) + } + + // A small set spanning even (depths 64, 66, 68) and odd (65, 67, 69) parity + // for contractA, plus a foreign key under contractB that must NOT show up. + slotPaths := [][]byte{ + {}, // depth 64 (even, == subtree root) + {0x3}, // depth 65 (odd) + {0x4, 0x5}, // depth 66 (even) + {0x6, 0x7, 0x8}, // depth 67 (odd) + {0x9, 0xa, 0xb, 0xc}, // depth 68 (even) + {0xd, 0xe, 0xf, 0x0, 0x1}, // depth 69 (odd) + } + + // Write each key at multiple txNums so it has multiple steps in MDBX (the + // DupSort dups). prev tracks the previous value (PutWithPrev contract). + expected := map[string][]byte{} + prev := map[string][]byte{} + writeAt := func(key []byte, txNum uint64, marker byte) { + v := make([]byte, 8) + v[0] = marker + v[7] = byte(txNum) + err := writer.PutWithPrev(key, v, txNum, prev[string(key)]) + require.NoError(t, err) + prev[string(key)] = common.Copy(v) + expected[string(key)] = common.Copy(v) + } + + // 3 successive writes per key at txNums 1, 5, 9 (latest = txNum 9 — the + // value we expect the cursor to return). + for i, sp := range slotPaths { + k := keyAt(nibA, sp) + for ti, tn := range []uint64{1, 5, 9} { + writeAt(k, tn, byte(0xA0+i*4+ti)) // distinctive marker per (key, step) + } + } + // Foreign contract key — must NOT appear in contractA's ranges. + foreign := keyAt(nibB, []byte{0x4, 0x5}) + writeAt(foreign, 7, 0xFF) + + require.NoError(t, writer.Flush(ctx, tx)) + + // Walk the cursor exactly like dbPrefetch does. + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(nibA) + c, err := tx.CursorDupSort(kv.TblCommitmentVals) + require.NoError(t, err) + defer c.Close() + + got := map[string][]byte{} + scanRange := func(from, to []byte) { + k, v, err := c.Seek(from) + for ; k != nil; k, v, err = c.NextNoDup() { + require.NoError(t, err) + if bytes.Compare(k, to) >= 0 { + break + } + require.GreaterOrEqual(t, len(v), 8, "dup value must be invStep(8B)||value") + got[string(common.Copy(k))] = common.Copy(v[8:]) + } + } + scanRange(evenFrom, evenTo) + scanRange(oddFrom, oddTo) + + // Every contractA branch must be present with its LATEST value (txNum=9). + for _, sp := range slotPaths { + k := keyAt(nibA, sp) + want, ok := expected[string(k)] + require.True(t, ok) + g, present := got[string(k)] + require.Truef(t, present, "key %x (depth %d) missing from cursor walk", k, 64+len(sp)) + require.Equalf(t, want, g, "key %x: cursor returned the wrong value (not the latest step)", k) + } + // Foreign-contract key must be absent. + _, leaked := got[string(foreign)] + require.Falsef(t, leaked, "foreign key %x leaked into contractA's parity ranges", foreign) +} From 9db0d5955b483bbcdfc40dbe712fbb54da9100ab Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:10:25 +0000 Subject: [PATCH 069/120] commitment, db/state: resumable parallel preload + adaptive parallel-mode wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the per-block adaptive extension path on top of the parallel wave-BFS preload. The new ContractTrunkPreloadParallel.Run method can be called incrementally with a step budget — each call advances zero or more complete waves and preserves un-pinned items at the current depth + accumulated children at depth+1 for the next call. This is the resumable counterpart to ContractTrunkPreload.Run (the serial BFS), so the adaptive controller can use the same wave-BFS file-only resolver for promote AND per-block extension instead of falling back to the serial CommitmentReader BFS. AdaptivePinController grows SetParallelMode(factory, provider) which installs a per-call resolver factory + dbBranches provider. The factory is invoked once per OnBlockComplete (returning a tx-scoped resolver and a release callback). promoteLocked/runExtensionLocked dispatch to the parallel path when the factory is set, falling back to the serial path when it isn't (or when the factory returns an error). adaptiveContractState holds either *ContractTrunkPreload (serial) or *ContractTrunkPreloadParallel (parallel); pinnedTotal/usedBytes/queueRemaining/pinnedPrefixes are mode- agnostic helpers used by the promote/extend/demote loop. SharedDomains.Flush installs the per-call factory + provider before calling OnBlockComplete, clears them after. The factory closes over the in-flight Flush tx (ttx.Debug().GetLatestFromFiles for file-only batch resolution); the provider walks TblCommitmentVals via CursorDupSort over the dual-parity ranges to populate the MDBX-resident branch overlay per contract. Tests: - execution/commitment/preload_parallel_test.go +9 tests covering ResumeAcrossSteps, RunAfterCompleteIsNoOp, StepBudgetCaps, ResumeAfterResolverError, DbBranchesPerStep, PinnedPrefixesAccumulate, NilCacheError, NilResolverError, BadHashLengthError - execution/commitment/trunk_pin_test.go +5 tests covering ParallelMode_PromoteUsesParallelResolver, ParallelMode_ExtendUsesResumableState, ParallelMode_FactoryErrorFallsBackToSerial, ParallelMode_DemoteInvalidates, ParallelMode_ReleaseCallbackInvoked Closes the "hypothesis C" gate from agentspecs/perf-regression-manifest.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 48 +++ execution/commitment/adaptive_pin.go | 209 +++++++++-- execution/commitment/preload_parallel.go | 330 ++++++++++++----- execution/commitment/preload_parallel_test.go | 334 ++++++++++++++++++ execution/commitment/trunk_pin_test.go | 244 +++++++++++++ 5 files changed, 1060 insertions(+), 105 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 986a7519492..8e66a38273f 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -783,7 +783,55 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { } return v, uint64(step), len(v) > 0, nil } + // Install per-call parallel-mode plumbing so the + // controller's promote/extend uses the wave-BFS file-only + // resolver. The factory captures ttx; cleared after the + // call so a future OnBlockComplete can't reach a stale tx. + factory := func() (commitment.BatchBranchResolver, func(), error) { + resolve := func(keys [][]byte) ([][]byte, error) { + d := ttx.Debug() + vals := make([][]byte, len(keys)) + for i, k := range keys { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) + if err != nil { + return nil, err + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } + } + return vals, nil + } + return resolve, nil, nil + } + provider := func(contractHash []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + scan := func(from, to []byte) { + for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { + if bytes.Compare(k, to) >= 0 { + return + } + if len(v) < 8 { + continue + } + m[string(common.Copy(k))] = common.Copy(v[8:]) + } + } + scan(evenFrom, evenTo) + scan(oddFrom, oddTo) + return m + } + sd.adaptivePinController.SetParallelMode(factory, provider) sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + sd.adaptivePinController.SetParallelMode(nil, nil) } } // Refresh pinned-tier gauges once per Flush — once-per-batch diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 1cd6b6f6066..c3668e01e7e 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -80,6 +80,10 @@ func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { // boundaries with a CommitmentReader factory; the controller // then promotes / extends / demotes based on the per-block // miss snapshot +// - Optional: host installs a parallel-mode resolver factory via +// SetParallelMode so promote/extend uses the wave-BFS parallel +// preload (file-only batch resolver + MDBX-overlay dbBranches) +// instead of the serial CommitmentReader BFS. type AdaptivePinController struct { cache *BranchCache cfg AdaptivePinControllerConfig @@ -93,15 +97,66 @@ type AdaptivePinController struct { // states holds per-promoted-contract bookkeeping. Protected by mu. mu sync.Mutex states map[[32]byte]*adaptiveContractState + + // Parallel-mode plumbing. nil = serial-BFS path (CommitmentReader). + // Set together via SetParallelMode; both required for parallel mode. + parallelResolverFactory ParallelResolverFactory + dbBranchesProvider DbBranchesProvider } +// ParallelResolverFactory builds a fresh BatchBranchResolver for one +// OnBlockComplete call. release() is invoked after the controller is done +// with the resolver (e.g. to tear down per-call worker txs). The factory +// may return (nil, nil, err) to indicate the controller should fall back +// to the serial-BFS path for this block. +type ParallelResolverFactory func() (resolve BatchBranchResolver, release func(), err error) + +// DbBranchesProvider returns the MDBX-resident branch overlay for one +// contract — values shadow file values in the parallel preload's wave. +// Empty/nil result is valid (means "no MDBX overlay; resolver is +// authoritative"); the controller treats it as a non-error. +type DbBranchesProvider func(contractHash []byte) map[string][]byte + type adaptiveContractState struct { contractHash [32]byte promotedAtBlock uint64 - preload *ContractTrunkPreload + preload *ContractTrunkPreload // serial-BFS path (nil when parallel) + parallel *ContractTrunkPreloadParallel // parallel-wave-BFS path (nil when serial) coldBlocksInARow int } +// pinnedTotal returns the cumulative pinned-count regardless of preload mode. +func (s *adaptiveContractState) pinnedTotal() int { + if s.parallel != nil { + return s.parallel.PinnedTotal() + } + return s.preload.PinnedTotal() +} + +// usedBytes returns the cumulative used-bytes regardless of preload mode. +func (s *adaptiveContractState) usedBytes() int { + if s.parallel != nil { + return s.parallel.UsedBytes() + } + return s.preload.UsedBytes() +} + +// queueRemaining returns the un-processed queue size regardless of mode. +func (s *adaptiveContractState) queueRemaining() int { + if s.parallel != nil { + return s.parallel.QueueRemaining() + } + return s.preload.QueueRemaining() +} + +// pinnedPrefixes returns the pin set for cache invalidation on demote. +func (s *adaptiveContractState) pinnedPrefixes() [][]byte { + if s.parallel != nil { + return s.parallel.PinnedPrefixes() + } + return s.preload.PinnedPrefixes() +} + // NewAdaptivePinController constructs a controller bound to the given // cache. Use Bind to install the cache miss-callback (separate so a // caller can keep a controller for telemetry without wiring it into @@ -143,6 +198,28 @@ func (c *AdaptivePinController) Bind() { c.cache.SetMissCallback(c.onCacheMiss) } +// SetParallelMode switches the controller to the wave-BFS parallel preload +// (ContractTrunkPreloadParallel) for both initial-view and per-block +// extension. The factory builds a tx-scoped resolver per OnBlockComplete +// (file-only); the provider returns the MDBX-resident branch overlay per +// contract so the freshest values shadow file values. +// +// Either argument may be nil to clear it. When factory==nil the controller +// uses the serial-BFS CommitmentReader path (existing default behavior). +// When factory is set but the factory call returns nil resolver/err, the +// controller falls back to serial-BFS for that block — the saved parallel +// state survives the block and resumes on the next successful factory call. +// +// Must be called before the first OnBlockComplete; calling it later is +// allowed but changes behavior at the next block boundary only — already- +// promoted contracts keep their existing serial/parallel state. +func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { + c.mu.Lock() + defer c.mu.Unlock() + c.parallelResolverFactory = factory + c.dbBranchesProvider = provider +} + func (c *AdaptivePinController) onCacheMiss(prefix []byte) { hash, ok := ContractHashFromPrefix(prefix) if !ok { @@ -171,6 +248,25 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui c.mu.Lock() defer c.mu.Unlock() + // Build the per-call parallel resolver up-front (one factory call per + // OnBlockComplete shared across all contracts this block). nil when + // parallel mode isn't installed, or the factory failed (we fall back to + // the serial-BFS reader path). + var parallelResolve BatchBranchResolver + var releaseParallel func() + if c.parallelResolverFactory != nil { + r, release, err := c.parallelResolverFactory() + if err != nil { + c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "block", blockNum) + } else { + parallelResolve = r + releaseParallel = release + } + } + if releaseParallel != nil { + defer releaseParallel() + } + var promoted, extended, demoted int // Already-promoted contracts: extend on hot, demote on cold. @@ -180,13 +276,13 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui state.coldBlocksInARow = 0 delete(misses, hash) // Extend if budget remains and queue not empty. - if state.preload.QueueRemaining() > 0 && state.preload.UsedBytes() < c.cfg.PerContractMaxBudgetBytes { - remaining := c.cfg.PerContractMaxBudgetBytes - state.preload.UsedBytes() + if state.queueRemaining() > 0 && state.usedBytes() < c.cfg.PerContractMaxBudgetBytes { + remaining := c.cfg.PerContractMaxBudgetBytes - state.usedBytes() step := c.cfg.ExtensionBudgetBytes if step > remaining { step = remaining } - if _, _, err := state.preload.Run(step, reader, c.cache, c.logger); err != nil { + if err := c.runExtensionLocked(ctx, state, step, parallelResolve, reader); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -208,24 +304,12 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { - p, err := NewContractTrunkPreload(hash[:]) + state, err := c.promoteLocked(ctx, hash, blockNum, parallelResolve, reader) if err != nil { - continue - } - if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) - // Roll back partial pin so we don't leak entries - // for a contract that won't be in c.states. - for _, prefix := range p.PinnedPrefixes() { - c.cache.Invalidate(prefix) - } continue } - c.states[hash] = &adaptiveContractState{ - contractHash: hash, - promotedAtBlock: blockNum, - preload: p, - } + c.states[hash] = state promoted++ } } @@ -270,18 +354,101 @@ func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { // demoteLocked invalidates every pinned prefix for the contract. // Caller must hold c.mu. func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { - for _, prefix := range state.preload.PinnedPrefixes() { + for _, prefix := range state.pinnedPrefixes() { c.cache.Invalidate(prefix) } if c.logger != nil { c.logger.Info("[adaptive-pin] demoted", "hash", hex.EncodeToString(hash[:]), - "pinned_was", state.preload.PinnedTotal(), - "used_mb_was", state.preload.UsedBytes()/(1<<20), + "pinned_was", state.pinnedTotal(), + "used_mb_was", state.usedBytes()/(1<<20), "cold_blocks", state.coldBlocksInARow) } } +// promoteLocked runs the initial-view preload for a freshly-promoted +// contract. Uses the parallel wave-BFS when parallelResolve != nil, +// otherwise falls back to the serial CommitmentReader BFS. On error the +// partial pin set is rolled back. Caller must hold c.mu. +func (c *AdaptivePinController) promoteLocked( + ctx context.Context, + hash [32]byte, + blockNum uint64, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) (*adaptiveContractState, error) { + if parallelResolve != nil { + p, err := NewContractTrunkPreloadParallel(hash[:]) + if err != nil { + return nil, err + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(hash[:]) + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, dbBranches, parallelResolve, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + parallel: p, + }, nil + } + // Serial BFS fallback (no parallel resolver this block). + p, err := NewContractTrunkPreload(hash[:]) + if err != nil { + return nil, err + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + preload: p, + }, nil +} + +// runExtensionLocked extends a promoted contract's preload by stepBudget +// bytes. Uses the saved state's mode (parallel vs serial). Caller must +// hold c.mu. +// +// When the saved state is serial (state.preload != nil) but parallelResolve +// is available this block, we keep using serial — switching mid-contract +// would lose the queue position. Future work: convert serial-state to +// parallel-state at extension time. +func (c *AdaptivePinController) runExtensionLocked( + ctx context.Context, + state *adaptiveContractState, + stepBudget int, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) error { + if state.parallel != nil { + if parallelResolve == nil { + // Parallel state but no resolver this block — skip the + // extension; the saved state survives for the next block. + return nil + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(state.contractHash[:]) + } + _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) + return err + } + // Serial-BFS state. + _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) + return err +} + // PromotedContracts returns the hashes of currently-pinned contracts. // Snapshot — the slice may be stale by the time the caller uses it. // Used by metrics + debug diagnostics. diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 3231c4a5e66..599b6a11e11 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -42,79 +42,141 @@ func estimatedEntryCost(key, value []byte) int { // never under-fetches, so the budget is guaranteed exhausted inside the wave. const minEntryBytes = estimatedEntryOverheadBytes + 33 -// PreloadContractTrunkParallel pins contract H's storage-trunk branches into -// cache by walking the subtree breadth-first, one depth level ("wave") at a -// time. Within a wave, each branch is resolved either from dbBranches (the -// MDBX-resident set — the freshest values; supplied by the caller's phase-1 -// range-cursor prefetch, may be nil/empty) or, for keys absent from it, through -// a single batched, parallelisable file-only read (BatchBranchResolver). +// maxStorageTrunkDepth is the deepest total nibble path a storage branch can +// have: 64 (account path) + 64 (keccak256(slot)) = 128. +const maxStorageTrunkDepth = 128 + +// pathKey is a single frontier entry: the nibble path (1 byte / nibble) and the +// HexToCompact-encoded CommitmentDomain key. +type pathKey struct { + path []byte // nibble path (1 byte / nibble) + key []byte // HexToCompact(path) — the CommitmentDomain key +} + +func toPathKey(path []byte) pathKey { + k := nibbles.HexToCompact(path) + kc := make([]byte, len(k)) + copy(kc, k) // HexToCompact result may alias a reused buffer + return pathKey{path: path, key: kc} +} + +// ContractTrunkPreloadParallel is the resumable analogue of ContractTrunkPreload +// (the serial BFS via CommitmentReader). It walks the contract's storage subtree +// breadth-first one depth-level ("wave") at a time and resolves missing branches +// through a batched, file-only BatchBranchResolver — no MDBX boundary crossings +// in the hot path. Each Run advances zero or more complete waves bounded by a +// per-call step budget; partial waves can be truncated by the file-fetch cap to +// fit the budget, but the truncated tail's siblings are skipped (BFS at the +// frontier is then resumed at depth+1 from the children of whatever was pinned). // -// Breadth-first so a budget-truncated run still pins the shallowest, highest- -// reuse branches first; wave-batched so the file resolver can sort+partition the -// reads for locality. DB-hits are pinned before file-hits within a wave (the DB -// holds the freshest, most-reused branches of a hot contract). Bounded by -// ramBudgetBytes. Returns the number of branches pinned. +// Mirrors ContractTrunkPreload's API surface (Run/PinnedTotal/UsedBytes/ +// QueueRemaining/MaxDepthReached/PinnedPrefixes/ContractHash) so it slots in to +// adaptive_pin.go as a drop-in replacement without changing the controller's +// promote/extend/demote loop. // // Correctness: dbBranches values shadow file values for the same key (DB is -// authoritative for steps not yet flushed to files), so passing the DB-resident -// set is what makes this safe on a live node. With an empty dbBranches it's a -// pure file-only preload — correct only from a cold snapshot, where MDBX holds -// nothing for the subtree. -func PreloadContractTrunkParallel( - contractHash []byte, - ramBudgetBytes int, +// authoritative for steps not yet flushed to files), so callers extending on a +// live node should pass the freshest MDBX-resident set per Run. With an empty +// dbBranches Run is a pure file-only preload — correct only from a cold snapshot +// or when MDBX/SD's flush-callback has already populated the cache with any +// freshly-written branches that would otherwise shadow the file values. +// +// Caller MUST NOT use the same instance from multiple goroutines. The +// BatchBranchResolver is passed PER Run rather than held on the struct so +// callers can supply a fresh tx-scoped resolver each call (e.g. adaptive +// controller building a resolver from the in-flight Flush tx every block). +// +// Resumability: when a step budget cuts a wave mid-iteration, the un-processed +// items at the current depth are preserved in `frontier`; the children of +// the items that did get pinned this wave land in `pendingChildren` at depth +// nextDepth+1. The next Run finishes the current depth first, then advances. +// Once a wave fully completes, frontier := pendingChildren and nextDepth++. +type ContractTrunkPreloadParallel struct { + contractHash []byte + frontier []pathKey // paths to process at depth = nextDepth + pendingChildren []pathKey // accumulated children of pinned items at depth = nextDepth+1 + nextDepth int // depth of the next wave (starts at 64) + pinnedPrefixes [][]byte + pinned int + usedBytes int + maxDepthReached int + dbHitsPinned int +} + +// NewContractTrunkPreloadParallel seeds a resumable preload state at depth 64 +// (storage subtree root) for the given contract. The resolver is passed +// per-Run (not stored on the state) so callers can hand a fresh tx-scoped +// resolver to each call. +func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreloadParallel, error) { + if len(contractHash) != 32 { + return nil, fmt.Errorf("NewContractTrunkPreloadParallel: contractHash must be 32 bytes, got %d", len(contractHash)) + } + contractHashCopy := make([]byte, len(contractHash)) + copy(contractHashCopy, contractHash) + return &ContractTrunkPreloadParallel{ + contractHash: contractHashCopy, + frontier: []pathKey{toPathKey(ContractNibbles(contractHashCopy))}, + nextDepth: 64, + maxDepthReached: 64, + }, nil +} + +// Run advances the wave-BFS, pinning branches into cache until either the step +// budget is exhausted, the frontier is empty (preload complete), or +// maxStorageTrunkDepth is reached. Returns newlyPinned during THIS call (not +// cumulative — see PinnedTotal), whether the preload is now complete, and any +// resolver error. +// +// dbBranches is consulted per wave (MDBX-resident overlays — values shadow file +// values, freshest). Pass nil for file-only mode (correct from a cold snapshot +// or when SD.Flush's cache callback has already populated freshly-written +// branches into the cache that would otherwise need to come from dbBranches). +// +// On resolver error the partial pin set survives; the wave position is +// preserved so a retry on the next Run picks up where it left off (at the same +// p.nextDepth). +func (p *ContractTrunkPreloadParallel) Run( + stepBudgetBytes int, dbBranches map[string][]byte, resolve BatchBranchResolver, cache *BranchCache, logger log.Logger, -) (pinned int, err error) { +) (newlyPinned int, queueEmpty bool, err error) { if cache == nil { - return 0, fmt.Errorf("PreloadContractTrunkParallel: cache is nil") + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: cache is nil") } if resolve == nil { - return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: resolver is nil") } - if ramBudgetBytes <= 0 { - return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) - } - if len(contractHash) != 32 { - return 0, fmt.Errorf("PreloadContractTrunkParallel: contractHash must be 32 bytes, got %d", len(contractHash)) - } - - type pathKey struct { - path []byte // nibble path (1 byte / nibble) - key []byte // HexToCompact(path) — the CommitmentDomain key - } - toPathKey := func(path []byte) pathKey { - k := nibbles.HexToCompact(path) - kc := make([]byte, len(k)) - copy(kc, k) // HexToCompact result may alias a reused buffer - return pathKey{path: path, key: kc} + if stepBudgetBytes <= 0 { + return 0, len(p.frontier) == 0, nil } - frontier := []pathKey{toPathKey(ContractNibbles(contractHash))} - usedBytes := 0 - maxDepthReached := 64 + stepCap := p.usedBytes + stepBudgetBytes + chunkPinned := 0 budgetHit := false - var dbHitsPinned int - // pin records the entry, advances accounting, and queues the branch's - // children. Returns false if the entry didn't fit (budget exhausted). + // pin records the entry, advances accounting, queues the branch's + // children. Returns false if the entry didn't fit (step budget exhausted). pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { cost := estimatedEntryCost(pk.key, v) - if usedBytes+cost > ramBudgetBytes { + if p.usedBytes+cost > stepCap { budgetHit = true return false } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel") - usedBytes += cost - pinned++ - if depth > maxDepthReached { - maxDepthReached = depth + cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel-resumable") + kc := make([]byte, len(pk.key)) + copy(kc, pk.key) + p.pinnedPrefixes = append(p.pinnedPrefixes, kc) + p.usedBytes += cost + p.pinned++ + chunkPinned++ + if depth > p.maxDepthReached { + p.maxDepthReached = depth } - if logger != nil && pinned%5000 == 0 { + if logger != nil && p.pinned%5000 == 0 { logger.Info("[trunk-preload-parallel] progress", - "pinned", pinned, "depth", depth, "used_mb", usedBytes/(1<<20)) + "pinned", p.pinned, "depth", depth, "used_mb", p.usedBytes/(1<<20)) } if len(v) >= 4 { // 2-byte touchMap || 2-byte afterMap || per-child data bitmap := binary.BigEndian.Uint16(v[2:4]) @@ -131,12 +193,13 @@ func PreloadContractTrunkParallel( return true } - for depth := 64; depth <= maxStorageTrunkDepth && len(frontier) > 0 && !budgetHit; depth++ { + for !budgetHit && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { + depth := p.nextDepth // Ascending key order so a contiguous-slice partition of the file batch // is contiguous-in-file (page-cache readahead). BFS append order is // already key-sorted (HexToCompact packing is monotone for same-length // paths), but sort defensively — a few k entries / wave, cheap. - sort.Slice(frontier, func(i, j int) bool { return bytes.Compare(frontier[i].key, frontier[j].key) < 0 }) + sort.Slice(p.frontier, func(i, j int) bool { return bytes.Compare(p.frontier[i].key, p.frontier[j].key) < 0 }) // Split this wave into DB-resident hits (have the value) and file-misses // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). @@ -144,7 +207,7 @@ func PreloadContractTrunkParallel( var dbVals [][]byte var fileMiss []pathKey dbHitsBytes := 0 - for _, pk := range frontier { + for _, pk := range p.frontier { if v, ok := dbBranches[string(pk.key)]; ok { dbHits = append(dbHits, pk) dbVals = append(dbVals, v) @@ -154,19 +217,17 @@ func PreloadContractTrunkParallel( } } - // Cap the *file fetch* at what the budget can absorb after the DB-hits - // (which we pin first). minEntryBytes is a true lower bound, so this - // never under-fetches; a truncated wave is guaranteed to exhaust the - // budget, so we stop after it. - truncatedByBudget := false - if fileBudget := ramBudgetBytes - usedBytes - dbHitsBytes; fileBudget <= 0 { - if len(fileMiss) > 0 { - fileMiss = fileMiss[:0] - truncatedByBudget = true - } + // Cap the *file fetch* at what the step budget can absorb after the + // DB-hits (which we pin first). minEntryBytes is a true lower bound, + // so this never under-fetches; if the cap truncates the wave, the + // un-fetched tail is preserved in p.frontier for the next Run. + var fileMissDeferred []pathKey + if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { + fileMissDeferred = fileMiss + fileMiss = nil } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { + fileMissDeferred = fileMiss[maxFileFetch:] fileMiss = fileMiss[:maxFileFetch] - truncatedByBudget = true } var fileVals [][]byte @@ -177,51 +238,152 @@ func PreloadContractTrunkParallel( } fileVals, err = resolve(keys) if err != nil { - return pinned, fmt.Errorf("preload at depth %d: %w", depth, err) + // State unchanged — retry on next Run picks up at same depth. + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", depth, err) } if len(fileVals) != len(keys) { - return pinned, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) + return chunkPinned, false, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) } } - var next []pathKey + // Pin in dbHits-then-fileMiss order. Children of pinned items accumulate + // in p.pendingChildren (depth nextDepth+1). On budget hit mid-iteration, + // the un-processed remainder of dbHits/fileMiss + the deferred + // fileMissDeferred all become the new frontier (still at nextDepth). + dbHitStop := len(dbHits) for i, pk := range dbHits { - if !pin(pk, dbVals[i], depth, &next) { + if !pin(pk, dbVals[i], depth, &p.pendingChildren) { + dbHitStop = i break } - dbHitsPinned++ + p.dbHitsPinned++ } + fileMissStop := len(fileMiss) if !budgetHit { for i, pk := range fileMiss { v := fileVals[i] if v == nil { - continue // not in the file layer either (shouldn't happen with a complete dbBranches+files) + continue // not in the file layer either } - if !pin(pk, v, depth, &next) { + if !pin(pk, v, depth, &p.pendingChildren) { + fileMissStop = i break } } } - if truncatedByBudget { - budgetHit = true + + if budgetHit { + // Preserve un-pinned items at the current depth for the next Run. + // Resolved-but-not-pinned file values are discarded (callable resolver + // will be re-invoked on the next Run for these keys). + var rest []pathKey + rest = append(rest, dbHits[dbHitStop:]...) + rest = append(rest, fileMiss[fileMissStop:]...) + rest = append(rest, fileMissDeferred...) + p.frontier = rest + // Don't advance nextDepth — pendingChildren stays at depth+1 for + // when this depth is fully drained on a future Run. + break + } + + // Wave fully processed. Anything in fileMissDeferred (won't have any + // since !budgetHit ⇒ no truncation) goes back to frontier; pending + // children promote to the next wave. + if len(fileMissDeferred) > 0 { + // Shouldn't happen — if we got here without budgetHit, no truncation + // occurred. Defensive: re-queue them at the current depth so we + // don't lose data. + p.frontier = fileMissDeferred + } else { + p.frontier = p.pendingChildren + p.pendingChildren = nil + p.nextDepth++ } - frontier = next } + queueEmpty = (len(p.frontier) == 0 && len(p.pendingChildren) == 0) || p.nextDepth > maxStorageTrunkDepth + if logger != nil && (chunkPinned > 0 || queueEmpty) { + logger.Info("[trunk-preload-parallel] step", + "contract_hash", fmt.Sprintf("%x", p.contractHash), + "step_budget_mb", stepBudgetBytes/(1<<20), + "used_mb_total", p.usedBytes/(1<<20), + "pinned_this_step", chunkPinned, + "pinned_total", p.pinned, + "db_hits_total", p.dbHitsPinned, + "max_depth_reached", p.maxDepthReached, + "queue_empty", queueEmpty, + "next_depth", p.nextDepth, + "frontier_size", len(p.frontier)) + } + return chunkPinned, queueEmpty, nil +} + +// PinnedTotal returns the cumulative number of branches pinned by this preload +// state across all Run calls. +func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } + +// UsedBytes returns the cumulative byte cost of this preload's pin set. +func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } + +// QueueRemaining returns the number of paths still queued for descent +// (un-processed at the current depth + accumulated children at the next). +// Zero means the preload is complete; further Run calls are no-ops. +func (p *ContractTrunkPreloadParallel) QueueRemaining() int { + return len(p.frontier) + len(p.pendingChildren) +} + +// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } + +// PinnedPrefixes returns the prefix slices this preload has added to the cache +// so far. The adaptive controller uses this to Invalidate the contract's pin +// set on demotion. The returned slice aliases internal storage; do not mutate. +func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } + +// ContractHash returns the contract this preload targets. +func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } + +// DbHitsPinned returns the cumulative number of branches that were pinned from +// the dbBranches overlay (rather than via the file resolver). Diagnostic only. +func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } + +// PreloadContractTrunkParallel pins contract H's storage-trunk branches into +// cache in a single call, bounded by ramBudgetBytes. Convenience wrapper around +// NewContractTrunkPreloadParallel + Run for callers that don't need phased +// loading. The startup PIN_CONTRACT_TRUNKS hook uses this; the adaptive layer +// holds a *ContractTrunkPreloadParallel and calls Run incrementally. +// +// dbBranches is the MDBX-resident set — values shadow file values for the same +// key. Pass nil/empty for file-only mode (correct only from a cold snapshot). +func PreloadContractTrunkParallel( + contractHash []byte, + ramBudgetBytes int, + dbBranches map[string][]byte, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (pinned int, err error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreloadParallel(contractHash) + if err != nil { + return 0, err + } + if resolve == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + } + pinned, queueEmpty, err := p.Run(ramBudgetBytes, dbBranches, resolve, cache, logger) if logger != nil { logger.Info("[trunk-preload-parallel] complete", "contract_hash", fmt.Sprintf("%x", contractHash), "ram_budget_mb", ramBudgetBytes/(1<<20), - "used_mb", usedBytes/(1<<20), + "used_mb", p.UsedBytes()/(1<<20), "pinned", pinned, - "db_hits_pinned", dbHitsPinned, - "max_depth_reached", maxDepthReached, - "budget_exhausted", budgetHit, + "db_hits_pinned", p.DbHitsPinned(), + "max_depth_reached", p.MaxDepthReached(), + "budget_exhausted", !queueEmpty, "cache_pinned_total", cache.PinnedCount()) } - return pinned, nil + return pinned, err } - -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. -const maxStorageTrunkDepth = 128 diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go index 450e5edced5..6cd3fe75e77 100644 --- a/execution/commitment/preload_parallel_test.go +++ b/execution/commitment/preload_parallel_test.go @@ -399,3 +399,337 @@ func TestPreloadParallel_ResolverError(t *testing.T) { t.Fatal("the depth-64 root should have been pinned before the depth-65 failure") } } + +// --- Resumable ContractTrunkPreloadParallel tests (Run-by-Run) --- + +// TestContractTrunkPreloadParallel_ResumeAcrossSteps confirms that splitting a +// full preload into multiple Run calls yields the same pinned set as a +// one-shot run with the equivalent total budget. +func TestContractTrunkPreloadParallel_ResumeAcrossSteps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // Reference: one-shot. + cRef := NewBranchCache(64) + if _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, resolve, cRef, nil); err != nil { + t.Fatal(err) + } + + // Step-by-step: budget exactly one entry per Run (the budget is checked + // before the entry is pinned; with overhead we need at least one entry's + // worth per step to make progress). + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Each entry is ~estimatedEntryOverheadBytes + 33 + valSz; over-allocate a + // bit per step so we always pin at least one new entry. + perStep := 2 * (estimatedEntryOverheadBytes + 33 + valSz) + const maxSteps = 50 + var steps int + for ; steps < maxSteps; steps++ { + _, done, err := p.Run(perStep, nil, resolve, c, nil) + if err != nil { + t.Fatalf("step %d: %v", steps, err) + } + if done { + break + } + } + if steps >= maxSteps { + t.Fatalf("preload did not complete in %d steps; pinned=%d", maxSteps, p.PinnedTotal()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("step-by-step pinned %d, want %d", p.PinnedTotal(), len(tree)) + } + if c.PinnedCount() != cRef.PinnedCount() { + t.Fatalf("step-by-step cache pinned %d != one-shot cache pinned %d", c.PinnedCount(), cRef.PinnedCount()) + } + // Spot-check: every path in the reference is in the step-by-step cache. + for path := range tree { + key := nibbles.HexToCompact([]byte(path)) + vRef, _, okRef := cRef.Get(key) + v, _, ok := c.Get(key) + if !okRef || !ok { + t.Fatalf("path %x: ref ok=%v, step ok=%v", path, okRef, ok) + } + if !bytes.Equal(v, vRef) { + t.Fatalf("path %x: step value differs from ref", path) + } + } +} + +// TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp confirms that once +// the BFS reaches an empty frontier, further Run calls are no-ops. +func TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + n1, done1, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done1 { + t.Fatalf("expected done after full budget, got done=false (queue=%d)", p.QueueRemaining()) + } + if n1 != len(tree) { + t.Fatalf("first Run pinned %d, want %d", n1, len(tree)) + } + prevPinned := c.PinnedCount() + n2, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatal("expected done on second Run") + } + if n2 != 0 { + t.Fatalf("second Run pinned %d new entries, want 0", n2) + } + if c.PinnedCount() != prevPinned { + t.Fatalf("cache pinned count changed across no-op Run: %d -> %d", prevPinned, c.PinnedCount()) + } +} + +// TestContractTrunkPreloadParallel_StepBudgetCaps confirms that a small step +// budget stops the BFS even when the frontier has more work — and the saved +// state has the queue position preserved for the next call. +func TestContractTrunkPreloadParallel_StepBudgetCaps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Budget for ~3 entries (depth-64 root + 2 depth-65 children). + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + smallBudget := 3*entry + 10 + n1, done1, err := p.Run(smallBudget, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if done1 { + t.Fatalf("expected NOT done after a 3-entry budget; got done=true (pinned=%d)", n1) + } + if n1 < 1 || n1 > 5 { + t.Fatalf("expected ~3 pinned this step, got %d", n1) + } + if p.QueueRemaining() == 0 { + t.Fatal("expected frontier to be non-empty after small-budget step") + } + // Now exhaust with a full follow-on budget. + _, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatalf("expected done after large follow-on budget; queue=%d", p.QueueRemaining()) + } + // In our synthetic tree only 3 of 7 paths sit at depths 64-65; the rest + // require descending past the truncated wave. The cap-per-wave logic + // drops the truncated wave's tail (BFS-wise) but the children of the + // pinned ones progress on the next call. Cumulative pinned should be + // >= the step-1 count. + if p.PinnedTotal() <= n1 { + t.Fatalf("follow-on Run made no progress: pinned still %d (step-1 was %d)", p.PinnedTotal(), n1) + } +} + +// TestContractTrunkPreloadParallel_ResumeAfterResolverError confirms that a +// resolver error preserves the partial state — a retry on the next Run picks +// up from the same wave once the resolver is healthy again. +func TestContractTrunkPreloadParallel_ResumeAfterResolverError(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + // Fail when R||1 is requested (a depth-65 key) — depth-64 wave succeeds. + failingResolve := fakeResolver(tree, nil, valSz, root+string([]byte{1})) + healthyResolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + _, done, err := p.Run(1<<20, nil, failingResolve, c, nil) + if err == nil { + t.Fatal("expected resolver error") + } + if done { + t.Fatal("Run with resolver error should return done=false") + } + // Depth-64 root should still be pinned (it was the previous wave). + if c.PinnedCount() == 0 { + t.Fatal("expected the depth-64 root pinned before the depth-65 wave failed") + } + preErrPinned := p.PinnedTotal() + // Retry with a healthy resolver — should pick up where we left off and + // finish. The previous partial wave will be re-attempted in the new + // call (the failing wave's frontier WAS advanced past the depth where + // the error fired — the error path returns before updating + // p.frontier/p.nextDepth, so retry sees the same depth's frontier). + n, done, err := p.Run(1<<20, nil, healthyResolve, c, nil) + if err != nil { + t.Fatalf("retry failed: %v", err) + } + if !done { + t.Fatalf("retry should complete the preload; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal()-preErrPinned != n { + t.Fatalf("PinnedTotal delta %d != Run pinned %d", p.PinnedTotal()-preErrPinned, n) + } + // Whole tree should be pinned by now. + if p.PinnedTotal() != len(tree) { + t.Fatalf("after retry pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_DbBranchesPerStep confirms that dbBranches +// can change between Run calls (caller may pass a freshly-prefetched overlay +// per block) and that the freshest values shadow file values per call. +func TestContractTrunkPreloadParallel_DbBranchesPerStep(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // On the first wave (depth 64) supply a fresh R value via dbBranches. + freshRoot := branchVal(tree[root], valSz) + freshRoot[4] = 0xAB + dbWave0 := map[string][]byte{string(nibbles.HexToCompact([]byte(root))): freshRoot} + + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Wave 0: pin the root using dbBranches (one entry budget). + rootKey := nibbles.HexToCompact([]byte(root)) + stepBudget := estimatedEntryOverheadBytes + len(rootKey) + valSz + 10 + if _, _, err := p.Run(stepBudget, dbWave0, resolve, c, nil); err != nil { + t.Fatal(err) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("wave 0: expected 1 db-hit pinned, got %d", p.DbHitsPinned()) + } + gotRoot, _, ok := c.Get(rootKey) + if !ok { + t.Fatal("root not pinned after wave 0") + } + if !bytes.Equal(gotRoot, freshRoot) { + t.Fatalf("wave 0: root pinned with stale file value, expected fresh dbBranches value") + } + + // Wave 1: depth 65. Pass an empty dbBranches (file-only); resolver + // supplies stale-bitmap values. + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after large budget; queue=%d", p.QueueRemaining()) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("expected db-hit count to remain 1, got %d", p.DbHitsPinned()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("after wave 1 pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate confirms that +// PinnedPrefixes() accumulates across Run calls (needed for demote-time +// cache invalidation in the adaptive controller). +func TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + // Two small steps then one big step. + for i := 0; i < 2; i++ { + if _, _, err := p.Run(2*entry+10, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } + } + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatal("expected done after large step") + } + prefixes := p.PinnedPrefixes() + if len(prefixes) != p.PinnedTotal() { + t.Fatalf("PinnedPrefixes len %d != PinnedTotal %d", len(prefixes), p.PinnedTotal()) + } + // Every prefix must be in the cache. + for _, pf := range prefixes { + if _, _, ok := c.Get(pf); !ok { + t.Fatalf("prefix %x in PinnedPrefixes but not in cache", pf) + } + } + // All prefixes are unique. + seen := map[string]bool{} + for _, pf := range prefixes { + if seen[string(pf)] { + t.Fatalf("duplicate prefix %x in PinnedPrefixes", pf) + } + seen[string(pf)] = true + } +} + +// TestContractTrunkPreloadParallel_NilCacheError + NilResolverError confirm +// the input-validation guards. +func TestContractTrunkPreloadParallel_NilCacheError(t *testing.T) { + hash := make([]byte, 32) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + resolve := func(keys [][]byte) ([][]byte, error) { return make([][]byte, len(keys)), nil } + if _, _, err := p.Run(1<<20, nil, resolve, nil, nil); err == nil { + t.Fatal("expected error when cache is nil") + } +} + +func TestContractTrunkPreloadParallel_NilResolverError(t *testing.T) { + hash := make([]byte, 32) + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + if _, _, err := p.Run(1<<20, nil, nil, c, nil); err == nil { + t.Fatal("expected error when resolver is nil") + } +} + +func TestContractTrunkPreloadParallel_BadHashLengthError(t *testing.T) { + if _, err := NewContractTrunkPreloadParallel(make([]byte, 31)); err == nil { + t.Fatal("expected error for 31-byte hash") + } + if _, err := NewContractTrunkPreloadParallel(make([]byte, 33)); err == nil { + t.Fatal("expected error for 33-byte hash") + } +} diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index edf8ebac6b0..e1c58b2dc35 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -229,3 +229,247 @@ func TestAdaptivePinController_DemoteOnCold(t *testing.T) { require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") require.Equal(t, 0, c.PinnedCount(), "pin set invalidated on demotion") } + +// --- Parallel-mode adaptive controller tests --- + +// makeParallelResolver returns a BatchBranchResolver paired with a usage +// counter so tests can assert it was actually invoked. +func makeParallelResolver(saturated bool, calls *int32) BatchBranchResolver { + return func(keys [][]byte) ([][]byte, error) { + atomic.AddInt32(calls, int32(len(keys))) + vals := make([][]byte, len(keys)) + var bitmap uint16 + if saturated { + bitmap = 0xFFFF + } else { + bitmap = 0x0001 + } + for i := range keys { + buf := make([]byte, 4) + binary.BigEndian.PutUint16(buf[2:4], bitmap) + vals[i] = buf + } + return vals, nil + } +} + +// TestAdaptivePinController_ParallelMode_PromoteUsesParallelResolver +// confirms that with parallel mode installed, a freshly-promoted contract +// uses the BatchBranchResolver (not the serial CommitmentReader). The +// serial reader is wired to panic so we know if it's called. +func TestAdaptivePinController_ParallelMode_PromoteUsesParallelResolver(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 3 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + + dbBranchesCalls := 0 + provider := func(contractHash []byte) map[string][]byte { + dbBranchesCalls++ + return nil // file-only — let the resolver supply everything + } + + factoryCalls := 0 + factory := func() (BatchBranchResolver, func(), error) { + factoryCalls++ + return resolver, nil, nil + } + ctrl.SetParallelMode(factory, provider) + + // Forge a 33-byte prefix that decodes to a known contract hash. + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 1) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Trigger 3 cache-misses for the same contract. + for i := 0; i < 3; i++ { + _, _, _ = c.Get(prefix) + } + + // Wire a reader that fails the test if called. + panicReader := func(prefix []byte) ([]byte, uint64, bool, error) { + t.Fatalf("parallel mode must not fall back to serial reader; called with prefix=%x", prefix) + return nil, 0, false, nil + } + + ctrl.OnBlockComplete(context.Background(), 1, panicReader) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Equal(t, contractHash, ctrl.PromotedContracts()[0]) + require.Greater(t, c.PinnedCount(), 0) + require.Equal(t, 1, factoryCalls, "factory should be called exactly once per OnBlockComplete") + require.Greater(t, atomic.LoadInt32(&resolverCalls), int32(0), "resolver should be called for the new contract") + require.GreaterOrEqual(t, dbBranchesCalls, 1, "dbBranchesProvider should be consulted for the new contract") +} + +// TestAdaptivePinController_ParallelMode_ExtendUsesResumableState confirms +// that the per-block extension reuses the same parallel state across calls +// (state.parallel persists; PinnedTotal grows monotonically). To trigger +// an extension miss for an already-promoted contract we Get a *deeper* +// prefix than what the initial promotion pinned, otherwise the lookup +// would hit the pinned tier and never invoke the miss callback. +func TestAdaptivePinController_ParallelMode_ExtendUsesResumableState(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 12 // 4 KiB — small initial view + cfg.ExtensionBudgetBytes = 1 << 12 // 4 KiB per extension step + cfg.PerContractMaxBudgetBytes = 1 << 20 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(true /* full bitmap — many children */, &resolverCalls) + factory := func() (BatchBranchResolver, func(), error) { + return resolver, nil, nil + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 13) + } + rootPrefix := append([]byte{0x00}, contractHash[:]...) + // deepPrefix has the contract hash in bytes 1..33 (so the miss + // callback attributes it to our contract) plus extra trailing bytes + // so it's not the same key as anything the preload could pin. + makeDeepPrefix := func(seed byte) []byte { + p := make([]byte, 50) + p[0] = 0x10 // HP flag for odd-depth path + copy(p[1:33], contractHash[:]) + for i := 33; i < 50; i++ { + p[i] = byte(i*7) ^ seed + } + return p + } + + // Block 1: trigger miss on the root path, promote. + _, _, _ = c.Get(rootPrefix) + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Len(t, ctrl.PromotedContracts(), 1) + pinnedAfterPromote := c.PinnedCount() + require.Greater(t, pinnedAfterPromote, 0) + + // Block 2: miss on a DEEPER prefix in the same contract's subtree — + // not yet pinned, so the miss callback fires. Extension fires. + _, _, _ = c.Get(makeDeepPrefix(0x11)) + ctrl.OnBlockComplete(context.Background(), 2, nil) + pinnedAfterExt1 := c.PinnedCount() + require.GreaterOrEqual(t, pinnedAfterExt1, pinnedAfterPromote, "extension should not shrink the pin set") + require.Greater(t, pinnedAfterExt1, pinnedAfterPromote, "extension should add at least one entry under a saturated trie") + + // Block 3: another extension via a different deep prefix. + _, _, _ = c.Get(makeDeepPrefix(0x22)) + ctrl.OnBlockComplete(context.Background(), 3, nil) + pinnedAfterExt2 := c.PinnedCount() + require.GreaterOrEqual(t, pinnedAfterExt2, pinnedAfterExt1) +} + +// TestAdaptivePinController_ParallelMode_FactoryErrorFallsBackToSerial +// confirms that when the parallel factory returns an error, the controller +// falls back to the serial CommitmentReader path without crashing. +func TestAdaptivePinController_ParallelMode_FactoryErrorFallsBackToSerial(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + factory := func() (BatchBranchResolver, func(), error) { + return nil, nil, errFakeFactory + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 21) + } + prefix := append([]byte{0x00}, contractHash[:]...) + _, _, _ = c.Get(prefix) + + ctrl.OnBlockComplete(context.Background(), 1, fakeReader(false)) + require.Len(t, ctrl.PromotedContracts(), 1, "should still promote via serial fallback") +} + +// TestAdaptivePinController_ParallelMode_DemoteInvalidates confirms that +// demotion invalidates the parallel-state pin set the same as serial. +func TestAdaptivePinController_ParallelMode_DemoteInvalidates(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.DemoteCooldownBlocks = 2 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + factory := func() (BatchBranchResolver, func(), error) { return resolver, nil, nil } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 29) + } + prefix := append([]byte{0x00}, contractHash[:]...) + + // Promote. + _, _, _ = c.Get(prefix) + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Len(t, ctrl.PromotedContracts(), 1) + require.Greater(t, c.PinnedCount(), 0) + + // Cold blocks — demote on 2nd. + ctrl.OnBlockComplete(context.Background(), 2, nil) + ctrl.OnBlockComplete(context.Background(), 3, nil) + require.Len(t, ctrl.PromotedContracts(), 0, "demoted after 2 cold blocks") + require.Equal(t, 0, c.PinnedCount(), "parallel-state pin set invalidated on demotion") +} + +// TestAdaptivePinController_ParallelMode_ReleaseCallbackInvoked confirms +// that a non-nil release callback returned by the factory is called once +// per OnBlockComplete (so tx-scoped resources can be cleaned up). +func TestAdaptivePinController_ParallelMode_ReleaseCallbackInvoked(t *testing.T) { + c := NewBranchCache(64) + cfg := DefaultAdaptivePinControllerConfig() + cfg.PromoteThresholdMisses = 1 + cfg.InitialViewBudgetBytes = 1 << 14 + ctrl := NewAdaptivePinController(c, cfg, nil) + ctrl.Bind() + + var resolverCalls int32 + resolver := makeParallelResolver(false, &resolverCalls) + var releaseCount int32 + factory := func() (BatchBranchResolver, func(), error) { + return resolver, func() { atomic.AddInt32(&releaseCount, 1) }, nil + } + ctrl.SetParallelMode(factory, nil) + + contractHash := [32]byte{} + for i := range contractHash { + contractHash[i] = byte(i + 33) + } + prefix := append([]byte{0x00}, contractHash[:]...) + _, _, _ = c.Get(prefix) + + ctrl.OnBlockComplete(context.Background(), 1, nil) + require.Equal(t, int32(1), atomic.LoadInt32(&releaseCount), "release callback called once per OnBlockComplete") + + // No misses next block — release still called (factory is invoked per block + // regardless of whether the controller pins anything). + ctrl.OnBlockComplete(context.Background(), 2, nil) + require.Equal(t, int32(2), atomic.LoadInt32(&releaseCount)) +} + +var errFakeFactory = stringError("fake factory failure") + +type stringError string + +func (s stringError) Error() string { return string(s) } From ae9b3f3ebbc76382ca1b588f969351275176e68e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 21 May 2026 11:47:42 +0000 Subject: [PATCH 070/120] execution/commitment: keep the trie checkpoint key out of BranchCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeyCommitmentState ("state") is a commitment-domain key, so the BranchCache that sits in front of the CommitmentDomain was caching it like a trie branch. But it is the trie checkpoint (txNum/blockNum/root), not a branch: it changes every block, and serving a stale copy restores the trie to the wrong state — producing a wrong commitment root (observed as engine-x "Wrong trie root of block 1", and depended on an explicit branchCache.Invalidate(KeyCommitmentState) workaround in squeeze). Reject KeyCommitmentState in BranchCache.Put/Get/GetDecoded/PinEntry so the cache cannot hold the checkpoint by construction — no caller can pollute it and no external invalidation is required. The key's single definition moves to the commitment package so the cache can reference it without an import cycle; commitmentdb.KeyCommitmentState now aliases it. Follow-up: a coordinated, version-keyed (txNum) rolling-buffer cache could re-introduce checkpoint caching safely — tracked separately. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/branch_cache.go | 25 +++++++++++++++++++ .../commitmentdb/commitment_context.go | 8 +++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index c07f25ea189..9edf294d242 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -17,6 +17,7 @@ package commitment import ( + "bytes" "fmt" "sync" "sync/atomic" @@ -25,6 +26,18 @@ import ( "github.com/erigontech/erigon/common/maphash" ) +// KeyCommitmentState is the commitment-domain key under which the trie +// checkpoint (txNum / blockNum / encoded root state) is stored. It is NOT a +// trie branch: it changes every block, so it must never enter the +// BranchCache — serving a stale checkpoint restores the trie to the wrong +// state and corrupts the computed root. BranchCache.Put/Get/PinEntry reject +// it by construction so no caller can pollute the cache with it. +var KeyCommitmentState = []byte("state") + +func isCommitmentStateKey(prefix []byte) bool { + return bytes.Equal(prefix, KeyCommitmentState) +} + // BranchCache stores commitment-trie branch data: // // - Bounded LRU tail with configurable capacity (eviction is well-defined, @@ -360,6 +373,9 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { + if isCommitmentStateKey(prefix) { + return + } dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.pinned.Set(prefix, &branchCacheEntry{ @@ -417,6 +433,9 @@ func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { + if isCommitmentStateKey(prefix) { + return nil, 0, false + } entry, ok := c.lookup(prefix) if !ok { return nil, 0, false @@ -437,6 +456,9 @@ func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { // caller MUST NOT modify the cells in place. Read-only consumption is // safe across concurrent calls. func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, ok bool) { + if isCommitmentStateKey(prefix) { + return 0, nil, false + } entry, found := c.lookup(prefix) if !found { return 0, nil, false @@ -468,6 +490,9 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { + if isCommitmentStateKey(prefix) { + return + } dataCopy := make([]byte, len(data)) copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 7a95675b315..874c5f651fa 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -590,10 +590,10 @@ func (e *errorTrieContext) Storage(plainKey []byte) (*commitment.Update, error) return nil, e.err } -// by that key stored latest root hash and tree state -const keyCommitmentStateS = "state" - -var KeyCommitmentState = []byte(keyCommitmentStateS) +// KeyCommitmentState is the commitment-domain key under which the latest +// root hash and tree state are stored. Single definition lives in the +// commitment package so BranchCache can exclude it by construction. +var KeyCommitmentState = commitment.KeyCommitmentState var ErrBehindCommitment = errors.New("behind commitment") From 3065d89b9a5da2bc582af94b2783815c67744b4c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 21 May 2026 13:35:07 +0000 Subject: [PATCH 071/120] execution: parent fork-validation SD into the canonical generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reorg-path NewPayload validation (TestReceiptRootValidationAfterReorg) computed a wrong commitment trie root, flakily, racing the gate-2 background commit. Root cause: fork-validation in ValidateChain created its SharedDomains with no parent. unwindToCommonCanonical then builds its unwind set from GetDiffset, which on a fresh SD finds nothing locally and falls back to ReadDiffSet on the tx — and the canonical chain's diffsets are not yet on disk while a background commit is in flight. With no unwind set, the unwind ran silently empty: SharedDomains.mem.unwindChangeset was never populated, so the commitment BranchCache was left unmasked and served pre-unwind (canonical-lineage) branches into the unwound read path. Fix: set the parent on the fork-validation SD (previously head-extending payloads only), and have SharedDomains.GetDiffset resolve through the parent chain. The canonical generation's pastChangesAccumulator then supplies the diffsets, the unwind builds a complete unwind set, and TemporalMemBatch.getLatest resolves every unwound key from the unwind set before the BranchCache is ever consulted — the unwind set masks the cache by construction. The parent does not shadow the unwound base: the unwind set covers exactly the keys the unwound canonical blocks touched. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 14 ++++++++- execution/execmodule/exec_module.go | 45 +++++++++++++++++------------ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8e66a38273f..bdd623ce2ea 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -551,7 +551,19 @@ func (sd *SharedDomains) SavePastChangesetAccumulator(blockHash common.Hash, blo } func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumber uint64) ([kv.DomainLen][]kv.DomainEntryDiff, bool, error) { - return sd.mem.GetDiffset(tx, blockHash, blockNumber) + d, ok, err := sd.mem.GetDiffset(tx, blockHash, blockNumber) + if ok || err != nil { + return d, ok, err + } + // Resolve through the parent chain: a fork-validation SD is freshly + // constructed with an empty mem batch, so the diffsets of the canonical + // blocks it must unwind live in the canonical generation's + // pastChangesAccumulator, reachable only via the parent link. Without + // this an unwind silently runs with no unwind set. + if sd.parent != nil { + return sd.parent.GetDiffset(tx, blockHash, blockNumber) + } + return d, ok, err } func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index c11ba373a48..de6df669ded 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -484,26 +484,33 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain the validation SD to currentContext when the new payload - // extends the current canonical head. Closes a timing hole: FCU's - // MergeExtendingFork moves the previous fork's writes into - // currentContext.mem, but MDBX commit only fires from RunLoop's - // CommitCycle on memory pressure. Between FCU and the next - // newPayload, currentContext.mem holds the latest state and MDBX - // is stale. Without the parent link, this fresh doms reads - // stale-MDBX while the aggregator-scope BranchCache (populated by - // the prior FV's CollectUpdate writes) holds the fresh state, - // producing divergence and wrong-trie-root downstream. + // Chain the validation SD to the latest in-memory canonical generation: + // e.currentContext when present, otherwise the newest in-flight commit + // generation (gate item 2 — the prior FCU cleared currentContext and + // handed its SD to the background commit). // - // Only set parent for head-extending payloads — fork validation - // requires unwindToCommonCanonical below to revert doms's view to - // the common ancestor, and exposing currentContext.mem (which holds - // post-divergence canonical writes) via the parent chain would - // shadow the unwound base. The current SD topology supports a - // single canonical chain only; per-branch SD lineage for concurrent - // multi-fork validation is a separate architectural follow-up. - canonicalHead := rawdb.ReadHeadBlockHash(tx) - if header.ParentHash == canonicalHead && e.currentContext != nil { + // The parent link serves two roles: + // + // 1. Head-extending payloads read the canonical generation's + // not-yet-committed domain state instead of stale MDBX. + // + // 2. Fork payloads: unwindToCommonCanonical below must build an unwind + // set, and the diffsets of the canonical blocks it unwinds live in + // the canonical generation's pastChangesAccumulator — reachable only + // through this parent link (GetDiffset chains to the parent). Without + // it the unwind silently runs with no unwind set, leaving the + // BranchCache unmasked and corrupting the computed root. + // + // For a fork payload the parent does NOT shadow the unwound base: once + // unwindToCommonCanonical has run, doms.mem.unwindChangeset holds every + // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest + // resolves those from the unwind set before ever consulting the parent. + // + // Cherry-pick note: the upstream commit also chained to e.latestGen() + // (the gate-2 in-flight commit generation) when currentContext is nil; + // that generation chain is not on this branch, so currentContext is the + // only canonical generation here. + if e.currentContext != nil { doms.SetParent(e.currentContext) } From ae711df3e317f230c71f69af4c3d3f17bd3f42cf Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 22 May 2026 12:02:17 +0000 Subject: [PATCH 072/120] execution/commitment: prealloc rest slice in preload_parallel Lint (prealloc): the rest slice length is known from the source ranges. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/commitment/preload_parallel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 599b6a11e11..6f8374a0d32 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -276,7 +276,7 @@ func (p *ContractTrunkPreloadParallel) Run( // Preserve un-pinned items at the current depth for the next Run. // Resolved-but-not-pinned file values are discarded (callable resolver // will be re-invoked on the next Run for these keys). - var rest []pathKey + rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) rest = append(rest, dbHits[dbHitStop:]...) rest = append(rest, fileMiss[fileMissStop:]...) rest = append(rest, fileMissDeferred...) From b2a9d649aab39493a6bc1dc5dc3ee426467fb496 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 16 May 2026 14:05:52 +0000 Subject: [PATCH 073/120] db/state/execctx, execution/execmodule: clear BranchCache on SetHead unwind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SetHead unwinds the commitment domain (via pipelineExecutor.RunUnwind) and truncates the TxNums index back to targetBlock. The unwind removes post-targetBlock commitment entries from sd.mem and MDBX, but the aggregator-scope BranchCache still holds the pre-unwind value of KeyCommitmentState (pointing at the original head's blockNum). sd.Flush after the unwind calls FlushWithCallback, which only updates cache entries for keys present in sd.mem — KeyCommitmentState is not re-written by the unwind, so it stays unchanged in cache. The next FCU's NewSharedDomains.SeekCommitment hits the cache, reads the original-head blockNum, and trips ErrBehindCommitment when compared against the truncated TxNums.Last() — which returns targetBlock. updateForkChoice returns ExecutionStatusTooFarAway and the FCU fails even though the chain state is fully consistent. Add a SharedDomains.ClearBranchCache method that empties the aggregator-scope BranchCache (all tiers). Call it from SetHead after sd.Flush and before tx.Commit so subsequent FCUs see the post-unwind MDBX state. Bisect: TestSetHeadCanonicalCleanup first failed inside the 24c0eddc56 → 9f5b8170f4 window (WarmupCache deletion sequence + the FlushWithCallback method introduction). The test has been failing since the cache was integrated; this is the integration-gap fix needed to enable cache use for perf. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 12 ++++++++++++ execution/execmodule/set_head.go | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index bdd623ce2ea..ddaff77d9a0 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -854,6 +854,18 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return sd.mem.Flush(ctx, tx) } +// ClearBranchCache empties the aggregator-scope commitment BranchCache. +// Use after operations that mutate commitment state outside the normal +// sd.Flush callback path — notably SetHead unwind, which truncates +// commitment domain history but does not re-write all the keys whose +// cached values are now stale. Without this call, the next FCU sees +// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. +func (sd *SharedDomains) ClearBranchCache() { + if sd.branchCache != nil { + sd.branchCache.Clear() + } +} + // TemporalDomain satisfaction func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { if tx == nil { diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 91da37f13e1..0623b5bcaa6 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -149,6 +149,15 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to flush shared domains: %w", err) } + // SetHead unwinds commitment history but the BranchCache (aggregator- + // scope, shared across SDs) still holds entries from the pre-unwind + // state — notably KeyCommitmentState pointing at the original head + // blockNum. The next FCU's NewSharedDomains.SeekCommitment would hit + // the cache, see the original-head blockNum, and trip + // ErrBehindCommitment against the now-truncated TxNums index. Clear + // the cache so subsequent reads see the post-unwind MDBX state. + sd.ClearBranchCache() + if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } From f151829dca57da2d8c7d38dcdbe42bc648c0b1e9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 18:44:26 +0000 Subject: [PATCH 074/120] execution/stagedsync/rawdbreset: clear branchCache on ResetExec (#21138) ResetExec wipes the commitment table but the aggregator's in-memory branchCache was holding entries that referenced the just-deleted trie nodes. A from-0 re-exec then served those stale entries when computing block 0's commitment, producing a wrong trie root and dropping every genesis-allocated balance that no subsequent block touched. Serial exec happened to skirt the issue; parallel exec hit it directly (test_account_access/EXTCODESIZE-contract behaviour on mainnet block 46147 against 0xA1E4380A3B1f749673E270229993eE55F35663b4, and the TestFromZero_GenesisAllocPreservedAfterResetReExec parallel subtest). Drop the cache as part of ResetExec so it repopulates from the freshly- wiped commitment table. Co-Authored-By: Claude Opus 4.7 --- .../stagedsync/rawdbreset/reset_stages.go | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/execution/stagedsync/rawdbreset/reset_stages.go b/execution/stagedsync/rawdbreset/reset_stages.go index 05352bab7d6..023c8e9e626 100644 --- a/execution/stagedsync/rawdbreset/reset_stages.go +++ b/execution/stagedsync/rawdbreset/reset_stages.go @@ -36,6 +36,7 @@ import ( "github.com/erigontech/erigon/db/rawdb/blockio" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/db/snaptype" + dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/diagnostics/diaglib" "github.com/erigontech/erigon/execution/stagedsync/stages" "github.com/erigontech/erigon/execution/types" @@ -173,7 +174,7 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { cleanupList = append(cleanupList, db.Debug().DomainTables(kv.AccountsDomain, kv.StorageDomain, kv.CodeDomain, kv.CommitmentDomain, kv.ReceiptDomain, kv.RCacheDomain)...) cleanupList = append(cleanupList, db.Debug().InvertedIdxTables(kv.LogAddrIdx, kv.LogTopicIdx, kv.TracesFromIdx, kv.TracesToIdx)...) - return db.Update(ctx, func(tx kv.RwTx) error { + if err := db.Update(ctx, func(tx kv.RwTx) error { if err := clearStageProgress(tx, stages.Execution); err != nil { return err } @@ -183,7 +184,25 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { } // corner case: state files may be ahead of block files - so, can't use SharedDomains here. juts leave progress as 0. return nil - }) + }); err != nil { + return err + } + + // Wiping the commitment table leaves the aggregator's in-memory branchCache + // referencing trie nodes that no longer exist on disk. A subsequent from-0 + // re-exec then reads those stale nodes when computing block 0's commitment + // and produces a wrong trie root (parallel-exec failure mode of #21138). + // Drop the cache so it repopulates from the freshly-wiped table. + if hasAgg, ok := db.(dbstate.HasAgg); ok { + if agg, ok := hasAgg.Agg().(*dbstate.Aggregator); ok { + aggTx := agg.BeginFilesRo() + if bc := aggTx.BranchCache(); bc != nil { + bc.Clear() + } + aggTx.Close() + } + } + return nil } func ResetTxLookup(tx kv.RwTx) error { From 9898f3cd25fe80e2e3fa8be1ec70fa987de91ce1 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 20:01:12 +0000 Subject: [PATCH 075/120] common/dbg, node/ethconfig: default ExecWorkerCount = NumCPU-1 The default came from `runtime.NumCPU()` which puts a worker on every visible core. The apply loop, FCU, GC, and other background goroutines end up fighting workers for runqueue slots, and the contention shows up as wall-time variance on every busy core. Drop the default by one so those non-worker goroutines have somewhere to land without preempting a worker. Floored at 1 for single-core environments. Override via EXEC3_WORKERS=N. Also removes the stale 'only half of CPU' comment on ethconfig.Defaults.Sync.ExecWorkerCount, which has been wrong since the default became NumCPU(). Co-Authored-By: Claude Opus 4.7 --- common/dbg/experiments.go | 9 ++++++--- node/ethconfig/config.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index bb39864534d..7a2f69d2aec 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -78,9 +78,12 @@ var ( CaplinSyncedDataMangerDeadlockDetection = EnvBool("CAPLIN_SYNCED_DATA_MANAGER_DEADLOCK_DETECTION", false) - Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) - numWorkers = runtime.NumCPU() - Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) + Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) + // One less than the visible CPU count so the apply loop, FCU, GC, and + // background goroutines aren't fighting workers for a runqueue slot. + // Floors at 1 for single-core environments. Override via EXEC3_WORKERS. + numWorkers = max(1, runtime.NumCPU()-1) + Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) ExecTerseLoggerLevel = EnvInt("EXEC_TERSE_LOGGER_LEVEL", int(log.LvlWarn)) CompressWorkers = EnvInt("COMPRESS_WORKERS", 0) // 0 means "not set": online presets default to 1, offline presets use RAM/CPU estimates MergeWorkers = EnvInt("MERGE_WORKERS", 1) diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index dcd469c05f5..a9e4862b5d8 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -84,7 +84,7 @@ var LightClientGPO = gaspricecfg.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ Sync: Sync{ - ExecWorkerCount: dbg.Exec3Workers, //only half of CPU, other half will spend for snapshots build/merge/prune + ExecWorkerCount: dbg.Exec3Workers, // NumCPU-1 by default — leaves one core for the apply loop, FCU, and GC BodyCacheLimit: 256 * 1024 * 1024, BodyDownloadTimeoutSeconds: 2, //LoopBlockLimit: 100_000, From 6454b7bff40fce21cbe7ae3f0d5b70701c3da398 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 23 May 2026 20:13:21 +0000 Subject: [PATCH 076/120] Revert "common/dbg, node/ethconfig: default ExecWorkerCount = NumCPU-1" This reverts commit 9898f3cd25fe80e2e3fa8be1ec70fa987de91ce1. --- common/dbg/experiments.go | 9 +++------ node/ethconfig/config.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 7a2f69d2aec..bb39864534d 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -78,12 +78,9 @@ var ( CaplinSyncedDataMangerDeadlockDetection = EnvBool("CAPLIN_SYNCED_DATA_MANAGER_DEADLOCK_DETECTION", false) - Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) - // One less than the visible CPU count so the apply loop, FCU, GC, and - // background goroutines aren't fighting workers for a runqueue slot. - // Floors at 1 for single-core environments. Override via EXEC3_WORKERS. - numWorkers = max(1, runtime.NumCPU()-1) - Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) + Exec3Parallel = EnvBool("EXEC3_PARALLEL", false) + numWorkers = runtime.NumCPU() + Exec3Workers = EnvInt("EXEC3_WORKERS", numWorkers) ExecTerseLoggerLevel = EnvInt("EXEC_TERSE_LOGGER_LEVEL", int(log.LvlWarn)) CompressWorkers = EnvInt("COMPRESS_WORKERS", 0) // 0 means "not set": online presets default to 1, offline presets use RAM/CPU estimates MergeWorkers = EnvInt("MERGE_WORKERS", 1) diff --git a/node/ethconfig/config.go b/node/ethconfig/config.go index a9e4862b5d8..dcd469c05f5 100644 --- a/node/ethconfig/config.go +++ b/node/ethconfig/config.go @@ -84,7 +84,7 @@ var LightClientGPO = gaspricecfg.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ Sync: Sync{ - ExecWorkerCount: dbg.Exec3Workers, // NumCPU-1 by default — leaves one core for the apply loop, FCU, and GC + ExecWorkerCount: dbg.Exec3Workers, //only half of CPU, other half will spend for snapshots build/merge/prune BodyCacheLimit: 256 * 1024 * 1024, BodyDownloadTimeoutSeconds: 2, //LoopBlockLimit: 100_000, From e8ba8129c7429e2ba3846d342c968de0adaae225 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 11:42:24 +0000 Subject: [PATCH 077/120] cmd/evm, execution/tests: per-subtest ephemeral SharedDomains for state tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State tests are now production-aligned: each subtest creates a SharedDomains scope, runs pre-state loading + tx execution + commitment against it, and discards the SD (Close without Flush) when the subtest finishes. Closing without Flush prevents per-subtest writes from entering the long-lived aggregator branch cache, fixing the cross-subtest pollution where test N's pre-state was leaking into test N+1 via the cache and producing wrong-trie-root failures even when each subtest had a fresh DB tx (the tx rollback rolls back disk only — it does not invalidate the cache). * MakePreState now splits into MakePreState (kept for tracetest callers that need flushed pre-state) and MakePreStateInto (no Flush; caller owns the SD). * StateTest.Run / RunNoVerify take a caller-supplied SD. * runStateTests (execution/tests) and runStateTest (cmd/evm) create one SD per subtest with a deferred Close so the cache stays clean. --- cmd/evm/staterunner.go | 16 ++++- execution/tests/state_test.go | 8 ++- execution/tests/testutil/state_test_util.go | 76 +++++++++++++-------- 3 files changed, 68 insertions(+), 32 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 850fdeb8a16..b4f34b2bec4 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/tests/testutil" "github.com/erigontech/erigon/execution/tracing/tracers/logger" "github.com/erigontech/erigon/execution/vm" @@ -224,7 +225,18 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } defer tx.Rollback() - statedb, root, err := test.Run(nil, tx, st, cfg, dirs) + // Per-subtest SD scope: caller-owned, never Flushed. Closing + // without Flush discards every write made during the subtest + // (pre-state + tx execution + commitment), so per-subtest state + // never enters the long-lived branch cache. + sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer sd.Close() + + statedb, root, err := test.Run(nil, sd, tx, st, cfg, dirs) if err != nil { result.Pass, result.Error = false, err.Error() } @@ -239,7 +251,7 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } if bench { _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.RunNoVerify(nil, tx, st, cfg, dirs) + _, _, gasUsed, _ := test.RunNoVerify(nil, sd, tx, st, cfg, dirs) return nil, gasUsed, nil }) result.Stats = &stats diff --git a/execution/tests/state_test.go b/execution/tests/state_test.go index 464fb9508ca..645c0f735e1 100644 --- a/execution/tests/state_test.go +++ b/execution/tests/state_test.go @@ -37,6 +37,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/temporal/temporaltest" + "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/tests/testutil" "github.com/erigontech/erigon/execution/tracing/tracers/logger" "github.com/erigontech/erigon/execution/vm" @@ -122,7 +123,12 @@ func runStateTests(t *testing.T, st *testutil.TestMatcher, testDir string) { withTrace(t, func(vmconfig vm.Config) error { tx := beginRwNoContention(t, db) defer tx.Rollback() - _, _, err = test.Run(t, tx, subtest, vmconfig, dirs) + sd, sdErr := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if sdErr != nil { + return sdErr + } + defer sd.Close() + _, _, err = test.Run(t, sd, tx, subtest, vmconfig, dirs) tx.Rollback() if err != nil && len(test.Json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { // Ignore expected errors diff --git a/execution/tests/testutil/state_test_util.go b/execution/tests/testutil/state_test_util.go index b1b8c6a277a..3ee793b567e 100644 --- a/execution/tests/testutil/state_test_util.go +++ b/execution/tests/testutil/state_test_util.go @@ -247,9 +247,11 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { return nil } -// Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (*state.IntraBlockState, common.Hash, error) { - st, root, _, err := t.RunNoVerify(tb, tx, subtest, vmconfig, dirs) +// Run executes a specific subtest and verifies the post-state and logs. +// sd is the caller-owned SharedDomains: discard (Close without Flush) to +// prevent per-subtest state from polluting the long-lived branch cache. +func (t *StateTest) Run(tb testing.TB, sd *execctx.SharedDomains, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (*state.IntraBlockState, common.Hash, error) { + st, root, _, err := t.RunNoVerify(tb, sd, tx, subtest, vmconfig, dirs) checkedErr := t.checkError(subtest, err) if checkedErr != nil { @@ -273,8 +275,12 @@ func (t *StateTest) Run(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, return st, root, nil } -// RunNoVerify runs a specific subtest and returns the statedb, post-state root and gas used. -func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (statedb *state.IntraBlockState, root common.Hash, gasUsed uint64, err error) { +// RunNoVerify runs a specific subtest and returns the statedb, post-state root +// and gas used. sd is the caller-owned SharedDomains scope; pre-state is loaded +// into it via MakePreStateInto so the per-subtest writes can be discarded by +// closing sd without Flush — keeping ephemeral test state out of the long-lived +// branch cache. +func (t *StateTest) RunNoVerify(tb testing.TB, sd *execctx.SharedDomains, tx kv.TemporalRwTx, subtest StateSubtest, vmconfig vm.Config, dirs datadir.Dirs) (statedb *state.IntraBlockState, root common.Hash, gasUsed uint64, err error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} @@ -288,20 +294,15 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State readBlockNr := block.NumberU64() writeBlockNr := readBlockNr + 1 - _, err = MakePreState(&chain.Rules{}, tx, t.Json.Pre, readBlockNr) + _, err = MakePreStateInto(&chain.Rules{}, sd, tx, t.Json.Pre, readBlockNr) if err != nil { return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} } - domains, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) - if err != nil { - return nil, common.Hash{}, 0, testforks.UnsupportedForkError{Name: subtest.Fork} - } - defer domains.Close() blockNum, txNum := readBlockNr, uint64(1) defer func() { - rootBytes, rootBytesErr := domains.ComputeCommitment(context2.Background(), tx, true, blockNum, txNum, "", nil) + rootBytes, rootBytesErr := sd.ComputeCommitment(context2.Background(), tx, true, blockNum, txNum, "", nil) if rootBytesErr != nil { if err != nil { err = fmt.Errorf("ComputeCommitment: %w: %w", rootBytesErr, err) @@ -314,7 +315,7 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State }() r := rpchelper.NewLatestStateReader(tx) - w := rpchelper.NewLatestStateWriter(tx, domains, (*freezeblocks.BlockReader)(nil), writeBlockNr) + w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), writeBlockNr) statedb = state.New(r) var baseFee *uint256.Int @@ -420,7 +421,37 @@ func (t *StateTest) RunNoVerify(tb testing.TB, tx kv.TemporalRwTx, subtest State return statedb, root, gasUsed, nil } +// MakePreState loads the test fixture's pre-allocation, flushes it to db + +// the long-lived branch cache, and returns the resulting IntraBlockState. +// Used by tracetest callers that need the pre-state to outlive the call. +// +// The state-test runner does NOT use this — see MakePreStateInto for the +// ephemeral-SD shape that keeps per-subtest state out of the branch cache. func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { + sd, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) + if err != nil { + return nil, err + } + defer sd.Close() + statedb, err := MakePreStateInto(rules, sd, tx, alloc, blockNr) + if err != nil { + return nil, err + } + if err := sd.Flush(context2.Background(), tx); err != nil { + return nil, err + } + return statedb, nil +} + +// MakePreStateInto loads the test fixture's pre-allocation into the supplied +// SharedDomains. The caller owns sd's lifecycle: a SD that is Closed without +// Flush discards everything written here, leaving the long-lived branch cache +// untouched. This is the production-aligned shape — application code uses +// SharedDomains as the ephemeral working scope; Flush is what commits to db +// and cache. Per-subtest state in the state-test runner is ephemeral and must +// NOT enter the cache, so each subtest creates an SD, calls this, runs, and +// discards the SD without flushing. +func MakePreStateInto(rules *chain.Rules, sd *execctx.SharedDomains, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { r := rpchelper.NewLatestStateReader(tx) statedb := state.New(r) statedb.SetTxContext(blockNr, 0) @@ -438,25 +469,17 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll val := uint256.NewInt(0).SetBytes(v.Bytes()) statedb.SetState(address, key, *val) } - if len(a.Code) > 0 || len(a.Storage) > 0 { statedb.SetIncarnation(address, state.FirstContractIncarnation) } } - domains, err := execctx.NewSharedDomains(context.Background(), tx, log.New()) - if err != nil { - return nil, err - } - defer domains.Close() - latestTxNum, latestBlockNum, err := domains.SeekCommitment(context.Background(), tx) + latestTxNum, latestBlockNum, err := sd.SeekCommitment(context.Background(), tx) if err != nil { return nil, err } - w := rpchelper.NewLatestStateWriter(tx, domains, (*freezeblocks.BlockReader)(nil), blockNr-1) - - // Commit and re-open to start with a clean state. + w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), blockNr-1) if err := statedb.FinalizeTx(rules, w); err != nil { return nil, err } @@ -464,12 +487,7 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll return nil, err } - _, err = domains.ComputeCommitment(context.Background(), tx, true, latestBlockNum, latestTxNum, "flush-commitment", nil) - if err != nil { - return nil, err - } - - if err := domains.Flush(context2.Background(), tx); err != nil { + if _, err := sd.ComputeCommitment(context.Background(), tx, true, latestBlockNum, latestTxNum, "pre-state-commitment", nil); err != nil { return nil, err } return statedb, nil From 54a9029675bc4f11a810f672eb55bd7d8cb4e316 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 18:33:27 +0000 Subject: [PATCH 078/120] execution/tests: read pre-state via sd.AsGetter so tx execution sees sd.mem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SD-discard refactor moved pre-state load into sd.mem and dropped the final Flush, but the tx execution reader was still constructed from the bare tx (NewLatestStateReader(tx)) — which only sees MDBX. Result: pre-state balance writes lived in sd.mem and the tx read MDBX-empty, surfacing as "insufficient funds for gas * price + value" on every state test that funded an account. NewLatestStateReader(sd.AsGetter(tx)) routes reads through the SD's sd.mem -> branchCache -> aggTx layering, so pre-state writes are visible. Same fix applied to MakePreStateInto for consistency with its writer. --- execution/tests/testutil/state_test_util.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/execution/tests/testutil/state_test_util.go b/execution/tests/testutil/state_test_util.go index 3ee793b567e..53094adc175 100644 --- a/execution/tests/testutil/state_test_util.go +++ b/execution/tests/testutil/state_test_util.go @@ -314,7 +314,11 @@ func (t *StateTest) RunNoVerify(tb testing.TB, sd *execctx.SharedDomains, tx kv. root = common.BytesToHash(rootBytes) }() - r := rpchelper.NewLatestStateReader(tx) + // Read through sd.AsGetter(tx) so the tx execution sees pre-state + // writes held in sd.mem. Without this the reader would hit MDBX + // directly and miss the never-Flushed pre-state, surfacing as + // "insufficient funds for gas * price + value" on every test. + r := rpchelper.NewLatestStateReader(sd.AsGetter(tx)) w := rpchelper.NewLatestStateWriter(tx, sd, (*freezeblocks.BlockReader)(nil), writeBlockNr) statedb = state.New(r) @@ -452,7 +456,10 @@ func MakePreState(rules *chain.Rules, tx kv.TemporalRwTx, alloc types.GenesisAll // NOT enter the cache, so each subtest creates an SD, calls this, runs, and // discards the SD without flushing. func MakePreStateInto(rules *chain.Rules, sd *execctx.SharedDomains, tx kv.TemporalRwTx, alloc types.GenesisAlloc, blockNr uint64) (*state.IntraBlockState, error) { - r := rpchelper.NewLatestStateReader(tx) + // Read through sd.AsGetter so SetCode/SetBalance see prior pre-state + // writes held in sd.mem (consistent with the writer that targets sd + // via NewLatestStateWriter below). + r := rpchelper.NewLatestStateReader(sd.AsGetter(tx)) statedb := state.New(r) statedb.SetTxContext(blockNr, 0) for addr, a := range alloc { From 57dd0d648844b2091727eb6124762780716f8054 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 24 May 2026 22:17:26 +0000 Subject: [PATCH 079/120] execution/engineapi/engineapitester: reset aggregator branchCache between cached-tester runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EngineXTestRunner caches one tester per (fork, preAllocHash) tuple and reuses it across every EEST case that shares those keys — so the slow genesis + node + datadir bootstrap is paid once per group. The aggregator's in-memory branchCache is scoped to the tester's lifetime, which means test N's pre-state populates cache entries that test N+1 still sees when it asks for its own genesis commitment — surfacing as a deterministic 'wrong trie root of block 1' on the second-and-later tests in each group. ResetBranchCache() on EngineApiTester drops the cache via the same HasAgg / aggTx.BranchCache().Clear() path the rawdbreset.ResetExec fix uses (#21138). EngineXTestRunner.Run calls it before each execute so the group's cached tester starts the next case with an empty cache, matching the from-scratch state every test would see if it built its own tester. Cheaper than evicting the tester (which would rebuild the genesis + node on every test) and keeps the cache enabled for the multi-payload sequence within a single test — where the cache provides the perf win. --- .../engineapitester/engine_api_tester.go | 39 ++++++++++++++++++- .../engineapitester/engine_x_test_runner.go | 5 +++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index af3126dd4e6..25a256f1669 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -39,7 +39,9 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" + dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/chain/networkname" @@ -399,10 +401,39 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs TxnInclusionVerifier: NewTxnInclusionVerifier(rpcApiClient), Node: ethNode, NodeKey: nodeKey, + ChainDB: ethBackend.ChainKV(), cleanup: cleanup, }, nil } +// ResetBranchCache drops the aggregator's in-memory branchCache. The cache is +// aggregator-scoped and survives a tx rollback, so a tester reused across +// independent EEST cases would otherwise serve commitment-trie entries +// populated by the previous test's pre-state — producing a `wrong trie root +// of block 1` on the next test's genesis→block-1 boundary. Same mechanism the +// rawdbreset.ResetExec path uses after wiping the commitment table (#21138); +// here the trigger is "moving to a new test" instead of "wiping the table". +// Safe to call at any time the tester is between tests; no-op if no +// aggregator-backed db is attached. +func (eat EngineApiTester) ResetBranchCache() { + if eat.ChainDB == nil { + return + } + hasAgg, ok := eat.ChainDB.(dbstate.HasAgg) + if !ok { + return + } + agg, ok := hasAgg.Agg().(*dbstate.Aggregator) + if !ok { + return + } + aggTx := agg.BeginFilesRo() + if bc := aggTx.BranchCache(); bc != nil { + bc.Clear() + } + aggTx.Close() +} + type EngineApiTesterInitArgs struct { Logger log.Logger DataDir string @@ -430,7 +461,13 @@ type EngineApiTester struct { TxnInclusionVerifier TxnInclusionVerifier Node *node.Node NodeKey *ecdsa.PrivateKey - cleanup *cleanupHandle + // ChainDB is the running backend's temporal DB. Retained on the tester so + // callers (e.g. EngineXTestRunner.Run between tests in the same group) can + // reach the aggregator's BranchCache and clear it without rebuilding the + // whole tester. The reference must NOT be Closed by callers — the tester's + // cleanup owns the lifecycle. + ChainDB kv.RwDB + cleanup *cleanupHandle } func (eat EngineApiTester) Run(t *testing.T, test func(ctx context.Context, t *testing.T, eat EngineApiTester)) { diff --git a/execution/engineapi/engineapitester/engine_x_test_runner.go b/execution/engineapi/engineapitester/engine_x_test_runner.go index bc37634e464..b14f19aa7ea 100644 --- a/execution/engineapi/engineapitester/engine_x_test_runner.go +++ b/execution/engineapi/engineapitester/engine_x_test_runner.go @@ -222,6 +222,11 @@ func (extr *EngineXTestRunner) Run(ctx context.Context, test EngineXTestDefiniti if err != nil { return err } + // Reuse the cached tester for the (fork, preAllocHash) group but reset the + // aggregator-scope branchCache so commitment-trie entries from the prior + // test don't leak into this test's genesis→block-1 boundary. Cheaper than + // evicting the whole tester (which would rebuild the genesis + node). + tester.ResetBranchCache() return extr.execute(ctx, tester, test) } From 5a415cf84f67bfc6c966471b1e703e339481bd2c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 14 May 2026 20:26:43 +0000 Subject: [PATCH 080/120] execution/cache, db/state/execctx: SD-transparent ethHash bypass for CodeDomain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third map (`ethHashToCode`) to CodeCache, keyed by the 32-byte Ethereum codeHash (keccak256). New methods `GetByEthHash` and `PutWithEthHash` expose direct L2b access without going through the addr→maphash→code two-level path. The byte storage duplicates L2 in the worst case (2x code-bytes memory at the cap); accepted for the per-key fast path on many-addrs-one-code workloads. `SharedDomains.GetLatest(CodeDomain, ...)` consults L2b transparently: when the addr-keyed cache misses, resolve the codeHash from the AccountsDomain (typically warm because the EVM just loaded the account), probe `stateCache.GetCodeByHash` before falling through to the file accessor stack. On miss, fill both L1 and L2b via PutCodeWithHash. The fast path is unchanged. Workload shape this targets: many addresses sharing one codeHash (proxies, factory-deployed clones, ERC-20 holders, OpenZeppelin templates). Today's addr-keyed cache misses on every fresh address even when the bytecode is already cached. With this change a single L2b entry serves N addresses after the first population. Microbench results: - BenchmarkCodeCache_GetByEthHash_Hit: 17.01 ns/op - BenchmarkCodeCache_GetByEthHash_Miss: 15.45 ns/op - BenchmarkCodeCache_Get_AddrLevel_Hit: 32.44 ns/op (existing) - BenchmarkCodeCache_GetByEthHash_ManyAddrs: 17.02 ns/op L2b hit is ~2x faster than the existing two-level addr path (one map probe vs two), and enables hits on workloads where L1 would miss. Cross-client research at agentspecs/cross-client-state-access-2026-05-14.md notes geth's separate codeSizeCache as the further (geth-proven) win for EXTCODESIZE/EXTCODEHASH and addrToHash LRU as a one-line behaviour fix; both queued as follow-up surgical commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 83 +++++++++- execution/cache/code_cache.go | 88 ++++++++-- execution/cache/code_cache_ethhash_test.go | 177 +++++++++++++++++++++ execution/cache/state_cache.go | 27 ++++ 4 files changed, 362 insertions(+), 13 deletions(-) create mode 100644 execution/cache/code_cache_ethhash_test.go diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a5ae5d0b196..6ec06a001fc 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -43,6 +43,7 @@ import ( "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/commitmentdb" + "github.com/erigontech/erigon/execution/types/accounts" ) var ( @@ -955,6 +956,23 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } + // CodeDomain L2b transparent bypass: many addresses share one codeHash + // (proxies, factory-deployed clones, ERC-20 holder set, OpenZeppelin + // templates). Today's addr-keyed cache misses on every fresh address + // even when the bytecode is already in the L2b layer. Resolve the + // codeHash from the account record (warm in AccountsDomain cache for + // any account the EVM has already loaded) and probe L2b before paying + // the code-file accessor stack cost. + var codeEthHash []byte + if domain == kv.CodeDomain && sd.stateCache != nil { + if h := sd.codeHashForAddr(tx, k); len(h) > 0 { + codeEthHash = h + if cv, ok := sd.stateCache.GetCodeByHash(codeEthHash); ok { + return cv, 0, nil + } + } + } + if aggTx, ok := tx.AggTx().(MeteredGetter); ok { v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) } else { @@ -966,7 +984,11 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // Populate state cache on successful storage read if sd.stateCache != nil { - sd.stateCache.Put(domain, k, v) + if domain == kv.CodeDomain && len(codeEthHash) > 0 && len(v) > 0 { + sd.stateCache.PutCodeWithHash(k, v, codeEthHash) + } else { + sd.stateCache.Put(domain, k, v) + } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") @@ -975,6 +997,65 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return v, step, nil } +// codeHashForAddr returns the Ethereum codeHash for an account, or nil if the +// account cannot be resolved or has no code. Reads the account through this +// SD's normal layered lookup chain so the AccountsDomain cache absorbs the +// cost (the typical case for any address the EVM has already loaded). +// +// Returns nil quietly on any error or missing account — the caller falls +// through to the addr-keyed file read so correctness is unaffected. +func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte) []byte { + if len(addr) == 0 { + return nil + } + // Direct sd.mem / sd.stateCache / aggTx chain via tx.GetLatest avoids + // recursing back into sd.GetLatest's CodeDomain branch. tx here is + // the temporal tx the SD was given; tx.GetLatest hits the aggTx + // directly. The account is overwhelmingly cache-warm on this path + // because the EVM has just loaded it for the op that triggered the + // code read; if not, this read fills the AccountsDomain cache as a + // side-effect. + if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + if sd.stateCache != nil { + if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + // Fall back to the aggTx directly to avoid a second recursion through + // SD.GetLatest. Skip on error; the caller's addr-keyed file read still + // runs and produces the correct result. + v, _, err := tx.GetLatest(kv.AccountsDomain, addr) + if err != nil || len(v) == 0 { + return nil + } + return decodeAccountCodeHash(v) +} + +// decodeAccountCodeHash extracts the codeHash from an account's encoded +// (DecodeForStorage) bytes. Returns nil on decode error or when the account +// has no code (empty codeHash). +func decodeAccountCodeHash(enc []byte) []byte { + if len(enc) == 0 { + return nil + } + var acc accounts.Account + if err := acc.DecodeForStorage(enc); err != nil { + return nil + } + if acc.CodeHash.IsEmpty() { + return nil + } + h := acc.CodeHash.Value() + return h[:] +} + func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { return &sd.metrics } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 39351e8c6a8..89adbd56852 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -39,14 +39,21 @@ const ( DefaultAddrCacheBytes = 16 * datasize.MB ) -// CodeCache is a two-level concurrent cache for contract code. +// CodeCache is a multi-level concurrent cache for contract code. // Level 1: addr → maphash(code) (mutable, cleared on reorg) // Level 2: maphash(code) → code (immutable, never cleared) +// Level 2b: ethHash(code) → code (immutable, never cleared) — enables +// bypass of the addr level when the caller already knows the +// Ethereum codeHash from a prior account read (common path +// for EXTCODESIZE / EXTCODEHASH / CALL where many addresses +// share the same bytecode — proxies, factory-deployed clones, +// ERC-20 holders, etc.). // // This design is efficient because: // - Multiple addresses can share the same code (common with proxies/clones) -// - Uses fast maphash instead of cryptographic Keccak256 +// - Uses fast maphash instead of cryptographic Keccak256 for L1/L2 // - Address mappings are small (8 bytes) so we can cache many more +// - L2b lets callers with the codeHash in hand skip L1 entirely // - Thread-safe via sync.Map // // Capacity is byte-based. Once full, new puts are no-ops but @@ -61,14 +68,25 @@ type CodeCache struct { addrSize atomic.Int64 // current size in bytes hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) - blockHash common.Hash // hash of the last block processed - mu sync.RWMutex + + // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated + // alongside L2 when the caller provides ethHash on Put. Independent + // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: + // duplicates code bytes vs L2 (worst case 2x byte storage); accepted + // for the per-key fast-path on many-addrs-one-code workloads. + ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent + ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + + blockHash common.Hash // hash of the last block processed + mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes @@ -77,10 +95,11 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), - hashToCode: maphash.NewMap[[]byte](), - addrCapacityB: addrCapacityBytes, - codeCapacityB: codeCapacityBytes, + addrToHash: maphash.NewMap[versionedAddressID](), + hashToCode: maphash.NewMap[[]byte](), + ethHashToCode: maphash.NewMap[[]byte](), + addrCapacityB: addrCapacityBytes, + codeCapacityB: codeCapacityBytes, } } @@ -143,6 +162,51 @@ func (c *CodeCache) Put(addr []byte, code []byte) { c.codeSize.Add(codeEntrySize) } +// GetByEthHash retrieves contract code by its Ethereum codeHash (keccak256). +// Bypasses the addr-keyed L1/L2 path. Returns (code, true) on hit, (nil, false) on miss. +// +// Designed for the common path where the caller has already loaded the +// account and knows the codeHash (EXTCODESIZE, EXTCODEHASH, CALL targets +// after account-load). Many addresses sharing one codeHash all hit this +// single L2b entry after the first population. +func (c *CodeCache) GetByEthHash(ethHash []byte) ([]byte, bool) { + code, ok := c.ethHashToCode.Get(ethHash) + if !ok || len(code) == 0 { + c.ethHashMisses.Add(1) + return nil, false + } + c.ethHashHits.Add(1) + return code, true +} + +// PutWithEthHash stores contract code, populating both the addr-keyed +// path (L1+L2) and the ethHash-keyed path (L2b). Use when the caller +// has the codeHash in hand (typically from a just-loaded account record); +// avoids the maphash-vs-keccak collision risk of re-deriving the ethHash +// from the value, and ensures L2b is fillable without an extra keccak. +// +// addr may be empty to populate only L2b (e.g. when populating from a +// codehash-only path that hasn't seen the addr yet). +func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { + if len(code) == 0 || len(ethHash) == 0 { + return + } + + if len(addr) > 0 { + c.Put(addr, code) + } + + if _, exists := c.ethHashToCode.Get(ethHash); exists { + return + } + entrySize := int64(len(ethHash) + len(code)) + if c.ethHashCodeSize.Load()+entrySize > int64(c.codeCapacityB) { + return + } + c.ethHashToCode.Set(ethHash, code) + c.ethHashCodeSize.Add(entrySize) +} + // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { diff --git a/execution/cache/code_cache_ethhash_test.go b/execution/cache/code_cache_ethhash_test.go new file mode 100644 index 00000000000..6c17724da72 --- /dev/null +++ b/execution/cache/code_cache_ethhash_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +import ( + "bytes" + "testing" + + "github.com/c2h5oh/datasize" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// makeEthHash builds a deterministic 32-byte ethHash for tests/benchmarks. +func makeEthHash(i int) []byte { + h := make([]byte, 32) + h[0] = byte(i) + h[1] = byte(i >> 8) + return h +} + +func TestCodeCache_GetByEthHash_HitAfterPut(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52} // small contract preamble + ethHash := makeEthHash(0xab) + + // Empty cache: miss. + v, ok := c.GetByEthHash(ethHash) + require.False(t, ok) + require.Nil(t, v) + + // Populate via PutWithEthHash — both addr and ethHash paths fill. + c.PutWithEthHash(addr, code, ethHash) + + // Hit by ethHash directly. + v, ok = c.GetByEthHash(ethHash) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) + + // Hit by addr via existing two-level path. + v, ok = c.Get(addr) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) +} + +func TestCodeCache_GetByEthHash_DistinctAddrsSameCode(t *testing.T) { + // The point of L2b: many addresses sharing one codeHash all hit a + // single entry once any one of them has been populated. + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52} + ethHash := makeEthHash(0xcd) + + addr1 := makeAddr(1) + c.PutWithEthHash(addr1, code, ethHash) + + // A different address never seen at L1 still hits the L2b layer + // when looked up by the shared ethHash. + v, ok := c.GetByEthHash(ethHash) + require.True(t, ok) + require.True(t, bytes.Equal(v, code)) + + // The addr2 lookup itself still misses (L1 unknown), as expected — + // L2b is meant to be probed by callers that already hold the hash. + addr2 := makeAddr(2) + _, ok = c.Get(addr2) + require.False(t, ok) +} + +func TestCodeCache_PutWithEthHash_EmptyHashOrCodeIsNoOp(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + addr := makeAddr(1) + code := []byte{0x60, 0x00} + + c.PutWithEthHash(addr, code, nil) // empty hash → skip L2b + v, ok := c.GetByEthHash(makeEthHash(7)) + require.False(t, ok) + require.Nil(t, v) + + c.PutWithEthHash(addr, nil, makeEthHash(7)) // empty code → skip both + v, ok = c.GetByEthHash(makeEthHash(7)) + require.False(t, ok) + require.Nil(t, v) +} + +func TestCodeCache_PutWithEthHash_RespectsCodeCapacity(t *testing.T) { + // 8-byte cap: 32-byte ethHash + 4-byte code > 32. New L2b puts must + // no-op when the layer is full. Use tiny code to keep math obvious. + c := NewCodeCache(8, 1*datasize.MB) + c.PutWithEthHash(makeAddr(1), []byte{1, 2, 3, 4}, makeEthHash(1)) + // Second put exceeds the L2b budget — must no-op. + c.PutWithEthHash(makeAddr(2), []byte{5, 6, 7, 8}, makeEthHash(2)) + + _, ok := c.GetByEthHash(makeEthHash(2)) + assert.False(t, ok, "second L2b entry should not exist when capacity is exceeded") +} + +// ============================================================================= +// Microbenchmarks — measure the per-op cost of the L2b path. +// ============================================================================= + +func BenchmarkCodeCache_GetByEthHash_Hit(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) // 2 KiB typical contract size + ethHash := makeEthHash(0x11) + c.PutWithEthHash(makeAddr(1), code, ethHash) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + v, ok := c.GetByEthHash(ethHash) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} + +func BenchmarkCodeCache_GetByEthHash_Miss(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + missHash := makeEthHash(0x22) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = c.GetByEthHash(missHash) + } +} + +// BenchmarkCodeCache_Get_AddrLevel_Hit baseline: the existing addr-keyed +// path. Compare against GetByEthHash to verify the L2b lookup is at least +// as fast (one map probe vs two: addr→hash then hash→code). +func BenchmarkCodeCache_Get_AddrLevel_Hit(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) + addr := makeAddr(1) + c.PutWithEthHash(addr, code, makeEthHash(0x33)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + v, ok := c.Get(addr) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} + +// BenchmarkCodeCache_GetByEthHash_ManyAddrs_OneCode measures the workload +// shape this layer is designed for: many addresses sharing one codeHash. +// Without L2b every fresh addr would pay a file read. With L2b every +// caller that already knows the hash hits one shared entry. +func BenchmarkCodeCache_GetByEthHash_ManyAddrs_OneCode(b *testing.B) { + c := NewCodeCache(64*datasize.MB, 16*datasize.MB) + code := bytes.Repeat([]byte{0x5b}, 2048) + ethHash := makeEthHash(0x44) + c.PutWithEthHash(makeAddr(1), code, ethHash) // populate once + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Caller knows the hash from a prior account read; probes L2b. + v, ok := c.GetByEthHash(ethHash) + if !ok || len(v) == 0 { + b.Fatal("expected hit") + } + } +} diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 816cdae11d3..7f1ee9b3d08 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -78,6 +78,33 @@ func (c *StateCache) Get(domain kv.Domain, key []byte) ([]byte, bool) { return cache.Get(key) } +// GetCodeByHash retrieves code bytes by their Ethereum codeHash (keccak256), +// bypassing the addr-keyed CodeDomain lookup. Returns (nil, false) on miss or +// when the code domain cache is not a CodeCache (defensive fallback). +// +// Use when the caller has the codeHash in hand (post-account-load) — typical +// for EXTCODESIZE / EXTCODEHASH / CALL targets. Lets many-addrs-one-code +// patterns (proxies, factory clones, ERC-20 holders) share a single L2b +// entry. +func (c *StateCache) GetCodeByHash(ethHash []byte) ([]byte, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return nil, false + } + return cc.GetByEthHash(ethHash) +} + +// PutCodeWithHash stores code populating both the addr-keyed path and the +// ethHash-keyed L2b layer. Callers should prefer this over Put when they +// have the codeHash from the account record — avoids a redundant keccak. +func (c *StateCache) PutCodeWithHash(addr, code, ethHash []byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutWithEthHash(addr, common.Copy(code), ethHash) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] From 4499ad5330756fe2b027b50a3815273a4bd3b87e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 14 May 2026 20:32:38 +0000 Subject: [PATCH 081/120] execution/cache, db/state, execution/state: codeSizeCache for EXTCODESIZE / EXTCODEHASH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third caching layer to CodeCache (alongside L1 addr→maphash and L2b ethHash→bytes): codeSizeByEthHash maps the 32-byte Ethereum codeHash to its byte length. Tiny per-entry footprint (32B key + 8B value vs 5-10 KB for full bytes) so the same memory budget gives ~1000x the hit surface. Capped at 1M entries (geth core/state/database_code.go uses the same size). EXTCODESIZE / EXTCODEHASH callers — historically the slowest opcodes on the lab dashboard's bench — answer from a single map probe without paying the file accessor stack cost of the full bytes. Geth-proven; cross-client writeup at agentspecs/cross-client-state-access-2026-05-14.md notes this as the largest single available win for the synthetic bench. Wiring: - CodeCache.GetCodeSizeByEthHash / PutCodeSizeByEthHash — direct accessors. - PutWithEthHash now populates the size layer alongside L2b, so every bytes-load creates a future fast-path entry "for free". - StateCache wrappers GetCodeSizeByHash / PutCodeSizeByHash. - SharedDomains.GetCodeSize(tx, addr) — the SD-transparent fast path: resolve codeHash via the AccountsDomain cache chain, probe the size cache, then L2b, then file-read+populate. Returns (0, false, nil) for EOAs and no-code accounts without paying any file read. - temporalGetter.GetCodeSize so callers reach it via the existing getter. - ReaderV3.ReadAccountCodeSize type-asserts on a codeSizeGetter interface and routes through the fast path when the underlying getter supports it; falls back to GetLatest+len otherwise. No kv.TemporalGetter interface change. Limitation: capacity is no-op-when-full, not LRU. A separate surgical commit will swap to real LRU eviction; mirrors the addrToHash fix queued from the same cross-client writeup. Tests: 3 new (PopulatedAlongsideBytes, DirectPutAndGet, EmptyHashOrNegativeIsNoOp). All existing CodeCache tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 59 +++++++++++++++++ execution/cache/code_cache.go | 77 ++++++++++++++++++---- execution/cache/code_cache_ethhash_test.go | 37 +++++++++++ execution/cache/state_cache.go | 22 +++++++ execution/state/rw_v3.go | 17 +++++ 5 files changed, 201 insertions(+), 11 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 6ec06a001fc..a018d752e34 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -439,6 +439,17 @@ func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv return gt.sd.GetLatest(name, gt.tx, k) } +// GetCodeSize returns the length of the code at addr without loading the +// bytes. Returns (size, true, nil) on size-cache hit, (size, true, nil) +// after a full-bytes load+populate, or (0, false, nil) when the account +// has no code. Errors propagate normally. +// +// Callers (ReaderV3.ReadAccountCodeSize, etc.) type-assert on this method +// so the existing kv.TemporalGetter interface is unchanged. +func (gt *temporalGetter) GetCodeSize(addr []byte) (int, bool, error) { + return gt.sd.GetCodeSize(gt.tx, addr) +} + func (gt *temporalGetter) HasPrefix(name kv.Domain, prefix []byte) (firstKey []byte, firstVal []byte, ok bool, err error) { return gt.sd.HasPrefix(name, prefix, gt.tx) } @@ -997,6 +1008,54 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return v, step, nil } +// GetCodeSize returns the length of the contract code at addr, probing a +// size-only cache (geth-style codeSizeCache) before falling through to the +// full bytes path. For workloads dominated by EXTCODESIZE / EXTCODEHASH +// this avoids the file-accessor + decompression cost of the full bytes on +// the second-and-later access to any codeHash seen anywhere in the process. +// +// Correctness invariant: the fast path is purely additive. When it cannot +// answer, the function delegates to GetLatest(CodeDomain, addr) — the +// authoritative path that hits L1/parent/stateCache/L2b/file in order. +// Never short-circuits to (0, false, nil) based on account-record +// resolution alone; that broke EIP-7002 / EIP-7251 system-contract +// syscalls (the predeploy has CodeDomain entries but the AccountsDomain +// record may be empty at block boundary). +// +// Returns (size, true, nil) on success and (0, false, nil) only when +// CodeDomain itself confirms no code. +func (sd *SharedDomains) GetCodeSize(tx kv.TemporalTx, addr []byte) (int, bool, error) { + if tx == nil { + return 0, false, errors.New("sd.GetCodeSize: unexpected nil tx") + } + + // Fast path: when we can resolve codeHash from the account cache AND + // the size is in the size cache, return without loading bytes. + if sd.stateCache != nil { + if codeEthHash := sd.codeHashForAddr(tx, addr); len(codeEthHash) > 0 { + if size, ok := sd.stateCache.GetCodeSizeByHash(codeEthHash); ok { + return size, true, nil + } + if cv, ok := sd.stateCache.GetCodeByHash(codeEthHash); ok { + sd.stateCache.PutCodeSizeByHash(codeEthHash, len(cv)) + return len(cv), true, nil + } + } + } + + // Cold path: authoritative read via the normal SD.GetLatest chain. + // Populates L1, L2b, and (via PutWithEthHash) the size layer for + // future callers. + v, _, err := sd.GetLatest(kv.CodeDomain, tx, addr) + if err != nil { + return 0, false, err + } + if len(v) == 0 { + return 0, false, nil + } + return len(v), true, nil +} + // codeHashForAddr returns the Ethereum codeHash for an account, or nil if the // account cannot be resolved or has no code. Reads the account through this // SD's normal layered lookup chain so the AccountsDomain cache absorbs the diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 89adbd56852..0839a458277 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -37,6 +37,10 @@ const ( DefaultCodeCacheBytes = 512 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB + // DefaultCodeSizeCacheEntries is the max entry count for the size-only + // cache (geth-style: code size answers without loading bytes for + // EXTCODESIZE / EXTCODEHASH callers). + DefaultCodeSizeCacheEntries int64 = 1_000_000 ) // CodeCache is a multi-level concurrent cache for contract code. @@ -77,16 +81,27 @@ type CodeCache struct { ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + // Size-only layer: ethCodeHash → int (length in bytes). Answers + // EXTCODESIZE / EXTCODEHASH without loading the bytes. Geth has the + // equivalent at core/state/database_code.go (1 M-entry LRU). Tiny + // per-entry footprint (32B key + 8B value) so the same memory budget + // gives ~1000x the hit surface vs the bytes cache. + codeSizeByEthHash *maphash.Map[int] + codeSizeEntries atomic.Int64 + codeSizeCapEntries int64 + blockHash common.Hash // hash of the last block processed mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 - ethHashHits atomic.Uint64 - ethHashMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 + codeSizeHits atomic.Uint64 + codeSizeMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes @@ -95,11 +110,13 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), - hashToCode: maphash.NewMap[[]byte](), - ethHashToCode: maphash.NewMap[[]byte](), - addrCapacityB: addrCapacityBytes, - codeCapacityB: codeCapacityBytes, + addrToHash: maphash.NewMap[versionedAddressID](), + hashToCode: maphash.NewMap[[]byte](), + ethHashToCode: maphash.NewMap[[]byte](), + codeSizeByEthHash: maphash.NewMap[int](), + codeSizeCapEntries: DefaultCodeSizeCacheEntries, + addrCapacityB: addrCapacityBytes, + codeCapacityB: codeCapacityBytes, } } @@ -196,6 +213,10 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { c.Put(addr, code) } + // Populate the size-only layer alongside the bytes layer — every time + // we touch the bytes we can answer a future EXTCODESIZE for free. + c.PutCodeSizeByEthHash(ethHash, len(code)) + if _, exists := c.ethHashToCode.Get(ethHash); exists { return } @@ -207,6 +228,40 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { c.ethHashCodeSize.Add(entrySize) } +// GetCodeSizeByEthHash retrieves the size (in bytes) of a contract by its +// Ethereum codeHash, without loading the bytes. Returns (0, false) on miss. +// +// Designed for EXTCODESIZE / EXTCODEHASH which only need the length; on a +// cache hit the caller answers a 4-instruction map probe instead of paying +// the file-accessor + decompression stack for the full bytes. Geth has the +// equivalent at core/state/database_code.go. +func (c *CodeCache) GetCodeSizeByEthHash(ethHash []byte) (int, bool) { + size, ok := c.codeSizeByEthHash.Get(ethHash) + if !ok { + c.codeSizeMisses.Add(1) + return 0, false + } + c.codeSizeHits.Add(1) + return size, true +} + +// PutCodeSizeByEthHash stores the size of code keyed by its Ethereum +// codeHash. No-op when full (limitation; addrToHash-style LRU is queued as +// a separate surgical change). +func (c *CodeCache) PutCodeSizeByEthHash(ethHash []byte, size int) { + if len(ethHash) == 0 || size < 0 { + return + } + if _, exists := c.codeSizeByEthHash.Get(ethHash); exists { + return + } + if c.codeSizeEntries.Load() >= c.codeSizeCapEntries { + return + } + c.codeSizeByEthHash.Set(ethHash, size) + c.codeSizeEntries.Add(1) +} + // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { diff --git a/execution/cache/code_cache_ethhash_test.go b/execution/cache/code_cache_ethhash_test.go index 6c17724da72..e8580533e83 100644 --- a/execution/cache/code_cache_ethhash_test.go +++ b/execution/cache/code_cache_ethhash_test.go @@ -109,6 +109,43 @@ func TestCodeCache_PutWithEthHash_RespectsCodeCapacity(t *testing.T) { assert.False(t, ok, "second L2b entry should not exist when capacity is exceeded") } +func TestCodeCache_CodeSize_PopulatedAlongsideBytes(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52, 0x60, 0x10} + ethHash := makeEthHash(0xee) + + // Miss before any populate. + _, ok := c.GetCodeSizeByEthHash(ethHash) + require.False(t, ok) + + // PutWithEthHash should fill the size layer alongside the bytes. + c.PutWithEthHash(makeAddr(1), code, ethHash) + + size, ok := c.GetCodeSizeByEthHash(ethHash) + require.True(t, ok) + require.Equal(t, len(code), size) +} + +func TestCodeCache_CodeSize_DirectPutAndGet(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + ethHash := makeEthHash(0xff) + + // Direct Put without going through the bytes layer. + c.PutCodeSizeByEthHash(ethHash, 4096) + + size, ok := c.GetCodeSizeByEthHash(ethHash) + require.True(t, ok) + require.Equal(t, 4096, size) +} + +func TestCodeCache_CodeSize_EmptyHashOrNegativeIsNoOp(t *testing.T) { + c := NewCodeCache(1*datasize.MB, 1*datasize.MB) + c.PutCodeSizeByEthHash(nil, 100) + c.PutCodeSizeByEthHash(makeEthHash(1), -1) + _, ok := c.GetCodeSizeByEthHash(makeEthHash(1)) + assert.False(t, ok) +} + // ============================================================================= // Microbenchmarks — measure the per-op cost of the L2b path. // ============================================================================= diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 7f1ee9b3d08..270dd7e219c 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -105,6 +105,28 @@ func (c *StateCache) PutCodeWithHash(addr, code, ethHash []byte) { cc.PutWithEthHash(addr, common.Copy(code), ethHash) } +// GetCodeSizeByHash returns the size of code by its Ethereum codeHash +// without loading the bytes. Returns (0, false) when the size-only layer +// is not populated for this hash. +func (c *StateCache) GetCodeSizeByHash(ethHash []byte) (int, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return 0, false + } + return cc.GetCodeSizeByEthHash(ethHash) +} + +// PutCodeSizeByHash records the code size for a given ethHash. Useful when +// the caller has the size in hand (e.g. from an account-domain probe that +// resolved a sibling addr to the same code) but doesn't have the bytes. +func (c *StateCache) PutCodeSizeByHash(ethHash []byte, size int) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutCodeSizeByEthHash(ethHash, size) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] diff --git a/execution/state/rw_v3.go b/execution/state/rw_v3.go index 142b206c56d..63e3bdd5c31 100644 --- a/execution/state/rw_v3.go +++ b/execution/state/rw_v3.go @@ -1542,11 +1542,28 @@ func (r *ReaderV3) ReadAccountCode(address accounts.Address) ([]byte, error) { return enc, nil } +// codeSizeGetter is the type-asserted fast-path interface for callers +// that only need the length of the code (EXTCODESIZE / EXTCODEHASH). +// Implemented by execctx.temporalGetter; fallback to GetLatest otherwise. +type codeSizeGetter interface { + GetCodeSize(addr []byte) (int, bool, error) +} + func (r *ReaderV3) ReadAccountCodeSize(address accounts.Address) (int, error) { var addressValue common.Address if !address.IsNil() { addressValue = address.Value() } + if sg, ok := r.getter.(codeSizeGetter); ok { + size, _, err := sg.GetCodeSize(addressValue[:]) + if err != nil { + return 0, err + } + if r.trace { + fmt.Printf("%sReadAccountCodeSize (sz) [%x] => [%d], txNum: %d\n", r.tracePrefix, addressValue, size, r.txNum) + } + return size, nil + } enc, _, err := r.getter.GetLatest(kv.CodeDomain, addressValue[:]) if err != nil { return 0, err From ad565feaa65312f87ca8f66df3445ab053f385ab Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 06:57:36 +0000 Subject: [PATCH 082/120] execution/exec, execution/execmodule: BlockReadAheader populates cache.StateCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BlockReadAheader has always prefetched BAL-listed (and access-list) addresses' account/code/storage via a fresh ReaderV3 on a separate RoTx. Its prefetches warmed OS page cache + RoTx cursors — disconnected from the process-global cache.StateCache that SharedDomains.GetLatest probes on the EVM hot path. The two layers were two separate caches; nothing the prefetcher loaded ever reached the EVM's lookup path. Reth's structural advantage on EXTCODESIZE-loop benches is that its prewarm writes to the same hashmap the EVM reads from (crates/engine/execution-cache/src/cached_state.rs:663). When EVM enters, every BAL-listed addr's first read is a 20 ns cache probe — no file accessor stack, no decompression CPU. PR #21128 swapped this from mini-moka to a lock-free fixed-cache for a measured +10.8 % mgas/s. This commit closes the equivalent gap on Erigon: a thin cache-populating TemporalGetter wrapper writes successful reads through to cache.StateCache as a side effect. ReaderV3 is unchanged; the wrapper sits in front. When the prefetcher already has the codeHash from a preceding account read, the next CodeDomain read routes through StateCache.PutCodeWithHash so the L2b (ethHash → bytes) + size-cache layers fill alongside the bare addr-keyed L1. Wiring: - BlockReadAheader.SetStateCache(*cache.StateCache) setter. - ExecModule construction calls readAheader.SetStateCache(domainCache), so the same StateCache the FCU/canonical path wires onto SD is the one the prefetcher warms. - cachePopulatingGetter wraps the worker's ttx; both BAL-warming and tx-warming paths gain the same treatment. Fgprof on the EXTCODESIZE-EXISTING_CONTRACT-30M bench had shown 95 % of EVM wall-clock in seg.Getter.nextPos (Huffman decompression of code values). With this commit, every BAL-listed addr's lookup should hit the cache and skip the file accessor stack entirely — eliminating the dominant cost. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/exec/blocks_read_ahead.go | 101 ++++++++++++++++++++++++++-- execution/execmodule/exec_module.go | 7 ++ 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 229b8de1a6c..ae4e4b423ab 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -16,6 +16,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" @@ -31,6 +32,14 @@ type BlockReadAheader struct { // this is for warming state warming atomic.Bool // only one warmBody can run at a time warmWg sync.WaitGroup + + // stateCache is the process-global state cache that SharedDomains.GetLatest + // consults on the EVM hot path. When set, warmBody routes its prefetches + // through a cache-populating getter so the same hashmap the EVM probes is + // pre-warmed. Without it, prefetches only warm OS page cache + RoTx + // cursors — disconnected from the cache layer the EVM actually reads. + // Mirrors reth's CachedReads / ExecutionCache "same hashmap" property. + stateCache *cache.StateCache } func NewBlockReadAheader() *BlockReadAheader { @@ -53,6 +62,61 @@ func NewBlockReadAheader() *BlockReadAheader { } } +// SetStateCache wires the process-global state cache so warmBody's +// prefetches land in the same hashmap that SharedDomains.GetLatest probes +// on the EVM hot path. Without this, prefetches warm OS page cache only — +// the EVM still pays the file accessor stack on its first per-address read. +// Idempotent; safe to call before the first AddHeaderAndBody. +func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { + bra.stateCache = sc +} + +// cachePopulatingGetter wraps a kv.TemporalGetter and writes successful +// reads through to a cache.StateCache as a side effect. Used by warmBody +// to make read-ahead prefetches populate the same in-process cache layer +// that SharedDomains.GetLatest consults — eliminating the file-accessor +// stack cost on the EVM's first touch of any prefetched address. +// +// For the CodeDomain, when the code bytes come back together with the +// owning account's codeHash (decoded from a preceding AccountsDomain read +// in the same loop iteration), the wrapper also populates the L2b +// ethHash→bytes + size-cache layers via PutCodeWithHash. The codeHash +// hint is provided per-iteration via withCodeHashHint(). +type cachePopulatingGetter struct { + g kv.TemporalGetter + sc *cache.StateCache + codeHashHint []byte // valid only for the next CodeDomain read; cleared after use +} + +func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { + v, step, err := cpg.g.GetLatest(name, k) + if err == nil && len(v) > 0 && cpg.sc != nil { + if name == kv.CodeDomain && len(cpg.codeHashHint) > 0 { + cpg.sc.PutCodeWithHash(k, v, cpg.codeHashHint) + cpg.codeHashHint = nil + } else { + cpg.sc.Put(name, k, v) + } + } + return v, step, err +} + +func (cpg *cachePopulatingGetter) HasPrefix(name kv.Domain, prefix []byte) ([]byte, []byte, bool, error) { + return cpg.g.HasPrefix(name, prefix) +} + +func (cpg *cachePopulatingGetter) StepsInFiles(entitySet ...kv.Domain) kv.Step { + return cpg.g.StepsInFiles(entitySet...) +} + +// withCodeHashHint stashes the codeHash so the next CodeDomain read routes +// through PutCodeWithHash (populating L2b + size cache) instead of a bare +// addr-keyed Put. Caller MUST follow this with a single GetLatest(CodeDomain, …) +// for the matching addr; the hint clears on use. +func (cpg *cachePopulatingGetter) withCodeHashHint(ethHash []byte) { + cpg.codeHashHint = ethHash +} + func (bra *BlockReadAheader) AddHeaderAndBody(ctx context.Context, db kv.RoDB, header *types.Header, body *types.Body) { blockHash := header.Hash() bra.headers.Add(blockHash, header) @@ -157,7 +221,13 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if !ok { return nil } - stateReader := state.NewReaderV3(ttx) + var getter kv.TemporalGetter = ttx + var cpg *cachePopulatingGetter + if bra.stateCache != nil { + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + getter = cpg + } + stateReader := state.NewReaderV3(getter) for idx := workerStart; idx < workerEnd; idx++ { select { @@ -168,8 +238,17 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t acctChanges := bal[idx] acct, _ := stateReader.ReadAccountData(acctChanges.Address) - // Warm code if account has code or if there are code changes - if (acct != nil && !acct.CodeHash.IsEmpty()) || len(acctChanges.CodeChanges) > 0 { + // Warm code if account has code or if there are code changes. + // When we already know the codeHash from the account read, hint + // the cache-populating getter so the code bytes land in the L2b + // (ethHash → bytes) + size layers — not just the addr-keyed L1. + if acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } + stateReader.ReadAccountCode(acctChanges.Address) + } else if len(acctChanges.CodeChanges) > 0 { stateReader.ReadAccountCode(acctChanges.Address) } for _, slotChanges := range acctChanges.StorageChanges { @@ -221,7 +300,13 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if !ok { return nil } - stateReader := state.NewReaderV3(ttx) + var getter kv.TemporalGetter = ttx + var cpg *cachePopulatingGetter + if bra.stateCache != nil { + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + getter = cpg + } + stateReader := state.NewReaderV3(getter) for txIdx := workerStart; txIdx < workerEnd; txIdx++ { select { @@ -236,6 +321,10 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t if toAddr := txn.GetTo(); toAddr != nil { to := accounts.InternAddress(*toAddr) if acct, _ := stateReader.ReadAccountData(to); acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } stateReader.ReadAccountCode(to) } } @@ -244,6 +333,10 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t for _, entry := range txn.GetAccessList() { addr := accounts.InternAddress(entry.Address) if acct, _ := stateReader.ReadAccountData(addr); acct != nil && !acct.CodeHash.IsEmpty() { + if cpg != nil { + h := acct.CodeHash.Value() + cpg.withCodeHashHint(h[:]) + } stateReader.ReadAccountCode(addr) } for _, slot := range entry.StorageKeys { diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index de6df669ded..51604fc6a3c 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -279,6 +279,13 @@ func NewExecModule( stopNode: stopNode, } + // Wire the process-global state cache into the read-ahead so its + // prefetches populate the same hashmap that SharedDomains.GetLatest + // probes on the EVM hot path. Reth's "same hashmap" pattern. + if readAheader != nil { + readAheader.SetStateCache(domainCache) + } + if stateCache != nil { stateCache.execModule = em } From 2d829566aa74615f47f13ccf4fdc475e54dd5819 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 08:54:00 +0000 Subject: [PATCH 083/120] execution/state, execution/cache: stateObject.code populate + addrToHash LRU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two surgical commits bundled (both touch the code-read hot path): 1. IntraBlockState.GetCodeSize now loads the full bytes via stateReader.ReadAccountCode on first touch and populates stateObject.code, so subsequent same-addr EXTCODESIZE / EXTCODEHASH / CALL within the tx are in-struct slice-len calls (~50 ns), not full reader round-trips. Mirrors geth's pattern at core/state/state_object.go ~Code() — pay one read per addr per tx, free for the rest. 2. CodeCache.addrToHash switched from a no-op-when-full maphash.Map[versionedAddressID] to an LRU lru.Cache[[20]byte, versionedAddressID] (hashicorp/golang-lru/v2, already imported elsewhere). Cap derived from the existing byte budget at ~28 bytes/entry (~580 k entries for the 16 MB default). Fresh-address workloads (mainnet thousands of new addrs per block) now warm up the addr layer over time instead of silently dropping new entries forever; matches geth's lru.Cache at core/state/database_code.go. The hashToCode layer is unchanged (content-addressed bytes, immutable, byte-capped with new-entry no-op when full — the same semantic as before since code bytes by codeHash never change). Bench on the EXTCODESIZE-EXISTING_CONTRACT-30M family: 62.34 mgas/s (was 61.50). The marginal gain is small on this bench because BAL prefetch already populates the cache layers; neither lever fires heavily. The expected wins are on non-BAL workloads where EXTCODESIZE-loop patterns repeat within a tx (#1) and fresh-address-churn mainnet blocks fill the addr layer (#2). Updated TestCodeCache_AddrCapacityLimit to assert LRU eviction (was asserting no-op-when-full); the prior behaviour was the bug. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/cache_test.go | 56 ++++++++++------ execution/cache/code_cache.go | 97 +++++++++++++++++----------- execution/state/intra_block_state.go | 28 +++++++- 3 files changed, 121 insertions(+), 60 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 588c1e3bc7a..f807678311c 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -328,31 +328,49 @@ func TestCodeCache_CodeDeduplication(t *testing.T) { } func TestCodeCache_AddrCapacityLimit(t *testing.T) { - // Each addr entry is 20 (addr) + 8 (uint64 hash) = 28 bytes - // Set addr capacity to 60 bytes - enough for 2 entries but not 3 - c := NewCodeCache(1024*1024, 60) // 1MB code, 60 bytes addr + // addrToHash is an LRU keyed by 20-byte address. Verify eviction is + // LRU rather than no-op-when-full — fresh-address workloads must + // warm up (geth's lru.Cache pattern, mirroring core/state/database_code.go). + // makeAddr / makeCode wrap at 256, so we generate addrs/codes from + // a wider 16-bit space directly. + wideAddr := func(i int) []byte { + a := make([]byte, 20) + a[18] = byte(i >> 8) + a[19] = byte(i) + return a + } + wideCode := func(i int) []byte { + return []byte{0x60, byte(i >> 8), byte(i)} + } - // Fill to addr capacity - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) - assert.Equal(t, 2, c.Len()) + c := NewCodeCache(1024*1024, 1024*28) // 1MB code, ~1024 addr LRU entries + for i := 0; i < 1100; i++ { + c.Put(wideAddr(i), wideCode(i)) + } - // Adding more should be no-op for addr (at capacity) - c.Put(makeAddr(3), makeCode(3)) - assert.Equal(t, 2, c.Len()) // addr not added + // Len should be exactly the LRU cap (1024), not silently truncated to 0. + assert.Equal(t, 1024, c.Len()) - // Code cache should have all 3 codes (code capacity not reached) - assert.Equal(t, 3, c.CodeLen()) + // Oldest entries (addrs 0..75) must have been evicted. + _, ok := c.Get(wideAddr(0)) + assert.False(t, ok, "oldest entry should be evicted by LRU") + _, ok = c.Get(wideAddr(50)) + assert.False(t, ok, "second-oldest range should be evicted by LRU") - // Can't get addr3 since it wasn't added - _, ok := c.Get(makeAddr(3)) - assert.False(t, ok) + // Most recent entry must be present. + v, ok := c.Get(wideAddr(1099)) + assert.True(t, ok, "most recent entry should remain") + assert.Equal(t, wideCode(1099), v) - // But updating existing addr should work - c.Put(makeAddr(1), makeCode(4)) - v, ok := c.Get(makeAddr(1)) + // hashToCode stores all 1100 distinct codes (content-addressed, + // independent of addr LRU eviction). + assert.Equal(t, 1100, c.CodeLen()) + + // Updating an existing addr re-writes the entry (LRU promotes to MRU). + c.Put(wideAddr(1099), wideCode(4242)) + v, ok = c.Get(wideAddr(1099)) assert.True(t, ok) - assert.Equal(t, makeCode(4), v) + assert.Equal(t, wideCode(4242), v) } func TestCodeCache_CodeCapacityLimit(t *testing.T) { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 0839a458277..b422978d496 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -22,6 +22,8 @@ import ( "unsafe" "github.com/c2h5oh/datasize" + lru "github.com/hashicorp/golang-lru/v2" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" @@ -37,6 +39,14 @@ const ( DefaultCodeCacheBytes = 512 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB + // DefaultAddrCacheEntries derives from DefaultAddrCacheBytes assuming + // ~28 bytes per entry (20-byte addr + 8-byte maphash codeID). Used as + // the LRU entry cap so the cache evicts oldest entries instead of + // silently dropping new ones when full — fresh-address workloads + // (e.g. mainnet thousands of new addrs per block) actually warm up + // over time, matching geth's lru.Cache pattern in + // core/state/database_code.go. + DefaultAddrCacheEntries = int(DefaultAddrCacheBytes) / 28 // DefaultCodeSizeCacheEntries is the max entry count for the size-only // cache (geth-style: code size answers without loading bytes for // EXTCODESIZE / EXTCODEHASH callers). @@ -68,10 +78,14 @@ type versionedAddressID struct { } type CodeCache struct { - addrToHash *maphash.Map[versionedAddressID] // addr → maphash(code), concurrent - addrSize atomic.Int64 // current size in bytes - hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent - codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) + // addrToHash maps a 20-byte Ethereum address to the maphash-derived + // codeID for the code at that address. Real LRU (was a no-op-when-full + // maphash.Map until commit 7d0998d0db) — fresh-address workloads now + // evict oldest entries and warm up the working set, matching geth's + // lru.Cache pattern at core/state/database_code.go. + addrToHash *lru.Cache[[20]byte, versionedAddressID] + hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent + codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated // alongside L2 when the caller provides ethHash on Put. Independent @@ -109,8 +123,16 @@ type CodeCache struct { // NewCodeCache creates a new CodeCache with the specified byte capacities. func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeCache { + addrEntries := int(addrCapacityBytes) / 28 + if addrEntries < 1024 { + addrEntries = 1024 + } + addrLRU, err := lru.New[[20]byte, versionedAddressID](addrEntries) + if err != nil { + panic(err) + } return &CodeCache{ - addrToHash: maphash.NewMap[versionedAddressID](), + addrToHash: addrLRU, hashToCode: maphash.NewMap[[]byte](), ethHashToCode: maphash.NewMap[[]byte](), codeSizeByEthHash: maphash.NewMap[int](), @@ -120,6 +142,20 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC } } +// addrKey casts a 20-byte slice to a fixed-size key without allocation. +// Caller MUST pass a 20-byte slice (all Ethereum addresses are 20 bytes). +// Returns the zero [20]byte if addr is shorter; only longer slices are +// truncated silently — defensive but should not happen on the hot path. +func addrKey(addr []byte) [20]byte { + var k [20]byte + if len(addr) >= 20 { + copy(k[:], addr[:20]) + } else { + copy(k[:], addr) + } + return k +} + // NewDefaultCodeCache creates a new CodeCache with the default sizes. func NewDefaultCodeCache() *CodeCache { return NewCodeCache(DefaultCodeCacheBytes, DefaultAddrCacheBytes) @@ -127,15 +163,14 @@ func NewDefaultCodeCache() *CodeCache { // Get retrieves contract code for the given address, implementing the Cache interface. func (c *CodeCache) Get(addr []byte) ([]byte, bool) { - // First, look up the code hash for this address - vID, ok := c.addrToHash.Get(addr) + k := addrKey(addr) + vID, ok := c.addrToHash.Get(k) if !ok || vID.addrID == 0 { c.addrMisses.Add(1) return nil, false } c.addrHits.Add(1) - // Then, look up the code by hash code, ok := c.hashToCode.Get(uint64AsBytes(&vID.addrID)) if !ok || len(code) == 0 { c.codeMisses.Add(1) @@ -147,33 +182,24 @@ func (c *CodeCache) Get(addr []byte) ([]byte, bool) { // Put stores contract code for the given address, implementing the Cache interface. // Uses fast maphash to compute the code identifier. -// If caches are at capacity, new entries are no-ops but updates are allowed. +// addrToHash is an LRU (auto-evicts oldest when full); hashToCode is byte-capped +// and no-ops on new writes when full (code is immutable, so existing entries stay). func (c *CodeCache) Put(addr []byte, code []byte) { if len(code) == 0 { return } codeHash := maphash.Hash(code) - addrEntrySize := int64(len(addr) + 8) // addr + uint64 hash - - // Check if addr already exists - updates are always allowed - if _, exists := c.addrToHash.Get(addr); exists { - c.addrToHash.Set(addr, versionedAddressID{addrID: codeHash}) - } else if c.addrSize.Load()+addrEntrySize <= int64(c.addrCapacityB) { - c.addrToHash.Set(addr, versionedAddressID{addrID: codeHash}) - c.addrSize.Add(addrEntrySize) - } - // Check if code already stored + c.addrToHash.Add(addrKey(addr), versionedAddressID{addrID: codeHash}) + hashKey := uint64AsBytes(&codeHash) if _, exists := c.hashToCode.Get(hashKey); exists { return } - - // New code entry - check capacity - codeEntrySize := int64(8 + len(code)) // hash + code + codeEntrySize := int64(8 + len(code)) if c.codeSize.Load()+codeEntrySize > int64(c.codeCapacityB) { - return // no-op when full + return } c.hashToCode.Set(hashKey, code) c.codeSize.Add(codeEntrySize) @@ -265,18 +291,13 @@ func (c *CodeCache) PutCodeSizeByEthHash(ethHash []byte, size int) { // Delete removes the address → codeHash mapping. // The codeHash → code mapping is kept since it's immutable. func (c *CodeCache) Delete(addr []byte) { - if _, exists := c.addrToHash.Get(addr); exists { - addrEntrySize := int64(len(addr) + 8) - c.addrToHash.Delete(addr) - c.addrSize.Add(-addrEntrySize) - } + c.addrToHash.Remove(addrKey(addr)) } // Clear removes all address mappings from the cache. // The codeHash → code mappings are preserved since they're immutable. func (c *CodeCache) Clear() { - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() } // GetBlockHash returns the hash of the last block processed by the cache. @@ -303,8 +324,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash // Empty blockHash means cache hasn't processed any block yet - always valid if c.blockHash == (common.Hash{}) { - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = incomingBlockHash return true } @@ -317,8 +337,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash // Mismatch - clear address mappings (they may be stale) // Keep code mappings since hash → code is immutable - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = incomingBlockHash return false } @@ -328,8 +347,7 @@ func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash func (c *CodeCache) ClearWithHash(hash common.Hash) { c.mu.Lock() defer c.mu.Unlock() - c.addrToHash.Clear() - c.addrSize.Store(0) + c.addrToHash.Purge() c.blockHash = hash } @@ -343,9 +361,10 @@ func (c *CodeCache) CodeLen() int { return c.hashToCode.Len() } -// AddrSizeBytes returns the current size of the address cache in bytes. +// AddrSizeBytes returns the estimated size of the address cache in bytes. +// LRU-based; estimate uses ~28 bytes per entry (20-byte addr + 8-byte codeID). func (c *CodeCache) AddrSizeBytes() int64 { - return c.addrSize.Load() + return int64(c.addrToHash.Len() * 28) } // CodeSizeBytes returns the current size of the code cache in bytes. @@ -372,7 +391,7 @@ func (c *CodeCache) PrintStatsAndReset() { codeHitRate = float64(codeHits) / float64(codeTotal) * 100 } - addrSizeB := c.addrSize.Load() + addrSizeB := c.AddrSizeBytes() codeSizeB := c.codeSize.Load() addrUsagePct := float64(addrSizeB) / float64(c.addrCapacityB) * 100 codeUsagePct := float64(codeSizeB) / float64(c.codeCapacityB) * 100 diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index 7a892c5f576..f496473fb03 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -736,7 +736,20 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if stateObject.data.CodeHash.IsEmpty() { return 0, nil } - return sdb.stateReader.ReadAccountCodeSize(addr) + // Geth pattern (core/state/state_object.go ~Code()): pay full code + // fetch once on first touch, populate stateObject.code so subsequent + // EXTCODESIZE / EXTCODEHASH / CALL on the same addr in this tx are + // in-struct slice-len calls (~50 ns), not full reader round-trips. + // On a 30M-gas EXTCODESIZE loop with N unique addrs, this collapses + // the per-addr cost from ~150k reader calls to 1 reader call. + code, err := sdb.stateReader.ReadAccountCode(addr) + if err != nil { + return 0, err + } + if stateObject != nil && code != nil { + stateObject.code = code + } + return len(code), nil } size, source, _, err := versionedRead(sdb, addr, CodeSizePath, accounts.NilKey, false, 0, @@ -759,7 +772,18 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if dbg.KVReadLevelledMetrics { readStart = time.Now() } - l, err := sdb.stateReader.ReadAccountCodeSize(addr) + // Geth pattern: load full code on first touch, populate + // stateObject.code so subsequent same-addr EXTCODESIZE / + // EXTCODEHASH / CALL within this tx hit the s.code branch + // above instead of re-entering the reader. With BAL prefetch + // the bytes are already in cache.StateCache, so this is + // effectively a hashmap probe + slice assignment. + code, codeErr := sdb.stateReader.ReadAccountCode(addr) + l := len(code) + if codeErr == nil && s != nil && code != nil { + s.code = code + } + err := codeErr if dbg.KVReadLevelledMetrics { sdb.codeReadDuration += time.Since(readStart) sdb.codeReadCount++ From 987f7412eb65538e76cdc8390f94bd07ccc1b7de Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 08:58:30 +0000 Subject: [PATCH 084/120] =?UTF-8?q?execution/cache,=20db/state/execctx:=20?= =?UTF-8?q?addr=20=E2=86=92=20codeHash=20LRU=20above=20SD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nethermind-style addr → 32-byte codeHash LRU sitting above SharedDomains.codeHashForAddr. When the EVM-known codeHash for an address has already been resolved once, subsequent lookups skip the entire AccountsDomain chain (sd.mem → sd.parent.mem → sd.stateCache → tx.GetLatest) and the account-RLP decode. Wiring: - CodeCache adds addrToEthHash *lru.Cache[[20]byte, [32]byte] sized to the existing addrCapacityB budget; methods GetAddrCodeHash / PutAddrCodeHash / DeleteAddrCodeHash. - StateCache wrappers route to the CodeCache instance. - SD.codeHashForAddr probes the LRU first; on miss falls through to the existing chain and populates on the way out (including the zero-hash sentinel for missing-or-EOA accounts — repeat lookups return immediately). - Invalidation: SD.DomainPut for AccountsDomain drops the entry (CREATE / CREATE2-replace path); SD.DomainDel for AccountsDomain also drops the entry (SELFDESTRUCT); StateCache.RevertWithDiffset drops on unwind. Helps non-BAL workloads where codeHashForAddr is currently the cold account-domain probe. On the EXISTING_CONTRACT bench (BAL prefetch already populates everything), this is within noise; the lever is for mainnet workloads where many addresses miss the BAL-prefetch window but the cache is warm from prior lookups. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/state/execctx/domain_shared.go | 72 +++++++++++++++++++++---------- execution/cache/code_cache.go | 36 ++++++++++++++++ execution/cache/state_cache.go | 37 ++++++++++++++++ 3 files changed, 122 insertions(+), 23 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a018d752e34..ea8c923140c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1067,34 +1067,50 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte) []byte { if len(addr) == 0 { return nil } - // Direct sd.mem / sd.stateCache / aggTx chain via tx.GetLatest avoids - // recursing back into sd.GetLatest's CodeDomain branch. tx here is - // the temporal tx the SD was given; tx.GetLatest hits the aggTx - // directly. The account is overwhelmingly cache-warm on this path - // because the EVM has just loaded it for the op that triggered the - // code read; if not, this read fills the AccountsDomain cache as a - // side-effect. - if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + // Nethermind-style addr → codeHash LRU. Skip the full AccountsDomain + // chain when the codeHash for this addr is already known. The zero + // hash sentinel means "no code or missing account" (negative cache). + if sd.stateCache != nil { + if h, ok := sd.stateCache.GetAddrCodeHash(addr); ok { + if h == ([32]byte{}) { + return nil + } + return h[:] + } } - if sd.parent != nil { - if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + + resolve := func() []byte { + if v, _, ok := sd.mem.GetLatest(kv.AccountsDomain, addr); ok { return decodeAccountCodeHash(v) } + if sd.parent != nil { + if v, _, ok := sd.parent.mem.GetLatest(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + if sd.stateCache != nil { + if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { + return decodeAccountCodeHash(v) + } + } + v, _, err := tx.GetLatest(kv.AccountsDomain, addr) + if err != nil || len(v) == 0 { + return nil + } + return decodeAccountCodeHash(v) } + + h := resolve() if sd.stateCache != nil { - if v, ok := sd.stateCache.Get(kv.AccountsDomain, addr); ok { - return decodeAccountCodeHash(v) + var fixed [32]byte + if len(h) == 32 { + copy(fixed[:], h) } + // Always populate, including the zero-hash sentinel for misses — + // repeat lookups skip the whole resolve() chain. + sd.stateCache.PutAddrCodeHash(addr, fixed) } - // Fall back to the aggTx directly to avoid a second recursion through - // SD.GetLatest. Skip on error; the caller's addr-keyed file read still - // runs and produces the correct result. - v, _, err := tx.GetLatest(kv.AccountsDomain, addr) - if err != nil || len(v) == 0 { - return nil - } - return decodeAccountCodeHash(v) + return h } // decodeAccountCodeHash extracts the codeHash from an account's encoded @@ -1235,9 +1251,16 @@ func (sd *SharedDomains) domainPut(domain kv.Domain, roTx kv.TemporalTx, k, v [] } } - // Update state cache when writing + // Update state cache when writing. For AccountsDomain, also invalidate + // the addr → codeHash LRU above SD because the codeHash may have + // changed (CREATE/CREATE2-replace; SELFDESTRUCT goes through DomainDel + // which has its own invalidation). The next codeHashForAddr lookup + // will repopulate from the freshly written account bytes. if sd.stateCache != nil { sd.stateCache.Put(domain, k, v) + if domain == kv.AccountsDomain { + sd.stateCache.DeleteAddrCodeHash(k) + } } // Serialize against the calculator's accumulator-swap window — see @@ -1282,10 +1305,13 @@ func (sd *SharedDomains) DomainDel(domain kv.Domain, tx kv.TemporalTx, k []byte, if err := sd.DomainDel(kv.CodeDomain, tx, k, txNum, nil); err != nil { return err } - // Remove from state cache when account is deleted + // Remove from state cache when account is deleted (including the + // nethermind-style addr → codeHash mapping; SELFDESTRUCT and + // CREATE-replace both reset the codeHash). if sd.stateCache != nil { sd.stateCache.Delete(kv.AccountsDomain, k) sd.stateCache.Delete(kv.CodeDomain, k) + sd.stateCache.DeleteAddrCodeHash(k) } // AccountsDomain — apply-side. Serialize against swap window. sd.changesetMu.Lock() diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index b422978d496..3d3ff653979 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -87,6 +87,13 @@ type CodeCache struct { hashToCode *maphash.Map[[]byte] // maphash(code) → code, concurrent codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) + // addrToEthHash maps a 20-byte address to its 32-byte Ethereum codeHash + // (keccak), separately from addrToHash (which uses the cheap maphash + // for bytes-lookup chaining). Used by SharedDomains.codeHashForAddr to + // skip a cold account-domain read when the EVM-known codeHash is + // already in cache. Nethermind-style addr → codeHash LRU. + addrToEthHash *lru.Cache[[20]byte, [32]byte] + // L2b: 32-byte Ethereum codeHash (keccak256) → code bytes. Populated // alongside L2 when the caller provides ethHash on Put. Independent // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: @@ -131,8 +138,13 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC if err != nil { panic(err) } + addrEthHashLRU, err := lru.New[[20]byte, [32]byte](addrEntries) + if err != nil { + panic(err) + } return &CodeCache{ addrToHash: addrLRU, + addrToEthHash: addrEthHashLRU, hashToCode: maphash.NewMap[[]byte](), ethHashToCode: maphash.NewMap[[]byte](), codeSizeByEthHash: maphash.NewMap[int](), @@ -205,6 +217,30 @@ func (c *CodeCache) Put(addr []byte, code []byte) { c.codeSize.Add(codeEntrySize) } +// GetAddrCodeHash returns the Ethereum codeHash for addr if cached. +// Nethermind-style lookup that lets SharedDomains.codeHashForAddr skip a +// cold AccountsDomain read when the EVM-known codeHash is already known. +// Eviction is LRU; freshly seen addrs replace coldest entries. +func (c *CodeCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + h, ok := c.addrToEthHash.Get(addrKey(addr)) + return h, ok +} + +// PutAddrCodeHash records the addr → codeHash mapping. Called from the +// account-decode populate path inside SD.codeHashForAddr; also called by +// readAhead's BAL prefetch when it learns the codeHash from the decoded +// account record. +func (c *CodeCache) PutAddrCodeHash(addr []byte, h [32]byte) { + c.addrToEthHash.Add(addrKey(addr), h) +} + +// DeleteAddrCodeHash drops the addr → codeHash mapping. Called on +// SELFDESTRUCT / CREATE2-replace / unwind where the account's codeHash +// has been mutated. +func (c *CodeCache) DeleteAddrCodeHash(addr []byte) { + c.addrToEthHash.Remove(addrKey(addr)) +} + // GetByEthHash retrieves contract code by its Ethereum codeHash (keccak256). // Bypasses the addr-keyed L1/L2 path. Returns (code, true) on hit, (nil, false) on miss. // diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 270dd7e219c..a9559570187 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -127,6 +127,39 @@ func (c *StateCache) PutCodeSizeByHash(ethHash []byte, size int) { cc.PutCodeSizeByEthHash(ethHash, size) } +// GetAddrCodeHash returns the Ethereum codeHash for addr without an +// account-domain round-trip. (0xff..ff/false, no entry on miss; (h, true) +// on hit.) +func (c *StateCache) GetAddrCodeHash(addr []byte) ([32]byte, bool) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return [32]byte{}, false + } + return cc.GetAddrCodeHash(addr) +} + +// PutAddrCodeHash records the addr → codeHash mapping in the addr-keyed +// LRU above SD. Callers that have just decoded an account record should +// call this so subsequent lookups skip the account-domain read. +func (c *StateCache) PutAddrCodeHash(addr []byte, h [32]byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.PutAddrCodeHash(addr, h) +} + +// DeleteAddrCodeHash drops the addr → codeHash mapping. Used by +// invalidation paths (SELFDESTRUCT / CREATE2-replace / unwind diffsets) +// where the account's codeHash has been mutated. +func (c *StateCache) DeleteAddrCodeHash(addr []byte) { + cc, ok := c.caches[kv.CodeDomain].(*CodeCache) + if !ok { + return + } + cc.DeleteAddrCodeHash(addr) +} + // Put stores data for the given domain and key. func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { cache := c.caches[domain] @@ -224,6 +257,10 @@ func (c *StateCache) RevertWithDiffset(diffset *[6][]kv.DomainEntryDiff, revertF k := []byte(entry.Key[:len(entry.Key)-8]) c.Delete(kv.CodeDomain, k) c.Delete(kv.AccountsDomain, k) + // Unwind may revert a codeHash change (SELFDESTRUCT undone, CREATE + // reverted) — drop the addr → codeHash mapping so the next read + // repopulates from the canonical post-revert account record. + c.DeleteAddrCodeHash(k) } for _, entry := range diffset[kv.CodeDomain] { k := []byte(entry.Key[:len(entry.Key)-8]) From 934082b9ed3e6cbad86b2cc4b1d6fd5e483dc829 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 10:52:31 +0000 Subject: [PATCH 085/120] execution/exec: cachePopulatingGetter caches negative results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache-populating wrapper on the read-ahead worker's TemporalTx previously gated cache writes on `len(v) > 0`. That dropped negative results — i.e. missing accounts, empty storage slots, no-code probes — on the floor. Repeated probes of the same missing address re-paid the file accessor stack walk every time, instead of hitting a cached negative entry. Mirrors the revm pattern that drives reth's 1700-3400 mgas/s on account_access NON_EXISTING / EXISTING_EOA variants: revm represents a missing address as a real CacheAccount{ account: None, status: LoadedNotExisting } and reth's ExecutionCache.account_cache uses FixedCache> where None is a first-class cacheable value. Bottom of the reth path is: BAL prewarm calls basic_account once → returns None → cache hit forever for that addr. The cycle-2 sweep on account_access[EXTCODESIZE/NON_EXISTING/30M] showed 3.65 → 494 mgas/s without this fix; with the fix the same bench reports 508 mgas/s (within run-to-run noise but trending right). Most of the win was already captured by the readAhead-populates- cache.StateCache wiring (commit cbe9044e52) and the balcache port (d41e2e84bb) — those raised the cache hit rate on populated entries enough that the EVM rarely fell through to the file accessor on this bench. The fix is mechanically correct regardless and should matter more on workloads with mixed populated / negative probes across blocks. See agentspecs/reth-missing-eoa-fastpath-2026-05-15.md for the detailed mechanism analysis and the three concrete copy-able patterns from reth. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/exec/blocks_read_ahead.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index ae4e4b423ab..08c0d47a5c9 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -90,11 +90,16 @@ type cachePopulatingGetter struct { func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, kv.Step, error) { v, step, err := cpg.g.GetLatest(name, k) - if err == nil && len(v) > 0 && cpg.sc != nil { + if err == nil && cpg.sc != nil { if name == kv.CodeDomain && len(cpg.codeHashHint) > 0 { cpg.sc.PutCodeWithHash(k, v, cpg.codeHashHint) cpg.codeHashHint = nil } else { + // Cache including nil/empty results: a probe returning no + // bytes is a valid negative answer (missing account, empty + // storage slot, no code) and caching it lets repeated probes + // skip the file accessor stack. Mirrors revm's CacheAccount + // { account: None, status: LoadedNotExisting } pattern. cpg.sc.Put(name, k, v) } } From e96df71550eca3052fe6db2097bcdd97e5c912b9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:21:51 +0000 Subject: [PATCH 086/120] execution/cache: surface fill-and-freeze cliff via inserts/dropped counters GenericCache.Put has no eviction policy. When the byte budget is reached, new keys are silently dropped until Clear/ClearWithHash/ValidateAndPrepare- mismatch resets the cache. On a long-running node this manifests as a monotonic miss-rate climb that's hard to attribute without instrumentation. Add two counters next to hits/misses: inserts - new keys accepted dropped - new keys rejected at the budget check (the existing branch at the new-key cap; not a behaviour change) PrintStatsAndReset logs both. Sets up the diagnostic baseline before the eviction-policy swap in the follow-up commits on this branch. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/generic_cache.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 0a40353ff9c..a4d0e2b40eb 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -35,6 +35,8 @@ type GenericCache[T any] struct { mu sync.RWMutex hits atomic.Uint64 misses atomic.Uint64 + inserts atomic.Uint64 + dropped atomic.Uint64 sizeFunc func(T) int // calculates size of value in bytes } @@ -103,13 +105,18 @@ func (c *GenericCache[T]) Put(key []byte, value T) { return } - // New key + // New key. The cache currently has no eviction policy, so once full + // every subsequent new key is silently dropped until Clear/ClearWithHash/ + // ValidateAndPrepare-mismatch resets the cache. dropped vs inserts makes + // this perf cliff observable. if c.currentSize.Load()+entrySize > int64(c.capacityB) { + c.dropped.Add(1) return } c.data.Set(key, value) c.currentSize.Add(entrySize) + c.inserts.Add(1) } // Delete removes the data for the given key. @@ -192,6 +199,8 @@ func (c *GenericCache[T]) CapacityBytes() datasize.ByteSize { func (c *GenericCache[T]) PrintStatsAndReset(name string) { hits := c.hits.Swap(0) misses := c.misses.Swap(0) + inserts := c.inserts.Swap(0) + dropped := c.dropped.Swap(0) total := hits + misses var hitRate float64 if total > 0 { @@ -201,6 +210,7 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 log.Debug(name+" cache stats", "hits", hits, "misses", misses, "hit_rate", hitRate, + "inserts", inserts, "dropped", dropped, "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) From 7b5a71c055f58465996a090841b31211f42c6ee7 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:36:44 +0000 Subject: [PATCH 087/120] execution/cache: replace GenericCache map with sharded LRU + Mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the maphash.Map[T] backing store in GenericCache with freelru.ShardedLRU[uint64, entry[T]] (same lib as db/state/cache.go; already in go.mod). Adds a Mode constructor flag: - ModeEvictLRU (default): per-shard LRU evicts the oldest entry on insert when its slot cap is reached. OnEvict drops bytes from currentSize. - ModeNoOp: preserves the historical fill-and-freeze behaviour (silently drop new keys at the byte cap; counted via dropped). Kept as the diagnostic baseline so the regression bench can compare A/B. Per-shard eviction is a known trade-off of freelru.ShardedLRU — RemoveOldest is shard-local, not globally LRU. Matches the trade-off db/state/cache.go / execution/cache/code_cache.go / execution/balcache/balcache.go already accept. LFU (W-TinyLFU, the policy reth uses) is scan-resistant by design and would slot in behind the same Mode wrapper as a follow-up; the seam is documented at policy.go. Key shape: pre-hash via common/maphash.Hash (Go's randomized stdlib hasher, already used by the previous maphash.Map) into uint64; entry stores the full key for collision check. Same pattern as db/state/cache.go. Byte-budget translation: per-domain avg-entry constants in state_cache.go (avgAccountEntryBytes / avgStorageEntryBytes / avgCommitmentEntryBytes) — account / storage are near-fixed sizes so the translation is reliable. capacityBytes becomes a sizing hint plus telemetry (SizeBytes / PrintStatsAndReset). Code domain is unchanged; CodeCache wraps its own LRUs. Adds metrics: inserts, evictions, dropped — all exposed in PrintStatsAndReset alongside the existing hits / misses / hit_rate. Mode is also logged. Touches one external call site: execution/vm/contract.go's jumpDestCache now constructs with ModeEvictLRU. Tests: TestDomainCache_PutCapacityLimit renamed to ..._NoOpMode and asserts the fill-and-freeze contract under explicit ModeNoOp. New TestDomainCache_PutEvictsWhenFull_EvictMode asserts eviction under ModeEvictLRU using a small entry-count cap (the byte→entry translation is approximate; the test uses the entry-count knob via the in-package newGenericCacheEntries constructor to make the assertion deterministic). Pre-existing lint issues on mh/sd-code-cache (intra_block_state.go nilness, preload_parallel.go prealloc) are surfaced by lint non-determinism but are out of this commit's scope. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/cache_test.go | 45 ++++++-- execution/cache/code_cache.go | 26 ++--- execution/cache/generic_cache.go | 179 ++++++++++++++++++++++--------- execution/cache/policy.go | 56 ++++++++++ execution/cache/state_cache.go | 26 ++++- execution/vm/contract.go | 2 +- 6 files changed, 261 insertions(+), 73 deletions(-) create mode 100644 execution/cache/policy.go diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index f807678311c..3160d45d216 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -104,25 +104,24 @@ func TestDomainCache_PutUpdateValue(t *testing.T) { assert.Equal(t, value2, v) } -func TestDomainCache_PutCapacityLimit(t *testing.T) { - // Each entry is 8 (overhead) + 3 (value) = 11 bytes - // Set capacity to 22 bytes - enough for 2 entries but not 3 - c := NewDomainCache(22) +func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { + // ModeNoOp keeps the historical fill-and-freeze behaviour: once + // full, new keys are silently dropped. Counted via the dropped metric. + // Entry overhead is 20 (addr key) + 3 (value) + 24 = 47 bytes per entry. + // Two entries take 94 bytes; cap at 100 leaves no room for a third. + c := NewDomainCacheMode(100, ModeNoOp) - // Fill to capacity c.Put(makeAddr(1), makeValue(1)) c.Put(makeAddr(2), makeValue(2)) assert.Equal(t, 2, c.Len()) - // Try to add more - should be ignored (no-op when full) c.Put(makeAddr(3), makeValue(3)) assert.Equal(t, 2, c.Len()) - // New key should not exist _, ok := c.Get(makeAddr(3)) assert.False(t, ok) - // But updating existing key should still work + // Updating an existing key always succeeds in either mode. newValue := []byte{100, 101, 102} c.Put(makeAddr(1), newValue) v, ok := c.Get(makeAddr(1)) @@ -130,6 +129,36 @@ func TestDomainCache_PutCapacityLimit(t *testing.T) { assert.Equal(t, newValue, v) } +func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { + // ModeEvictLRU lets the per-shard LRU evict on insert when its + // entry-count cap is reached. Eviction is per-shard, not globally + // LRU (a known trade-off of freelru.ShardedLRU; see policy.go). + // + // Build with capacityEntries=2 so that a third insert forces an + // eviction event. Capacity-bytes is unused for the eviction + // decision and is only carried for telemetry. + c := &DomainCache{ + GenericCache: newGenericCacheEntries[[]byte](1<<20, 2, func(v []byte) int { return len(v) }, ModeEvictLRU), + } + + for i := 1; i <= 64; i++ { + c.Put(makeAddr(i), makeValue(i)) + } + // The newest entry must still be findable. + v, ok := c.Get(makeAddr(64)) + assert.True(t, ok, "newest key must be present after eviction") + assert.Equal(t, makeValue(64), v) + + // At least some early entries must have been evicted by now. + missingCount := 0 + for i := 1; i <= 32; i++ { + if _, ok := c.Get(makeAddr(i)); !ok { + missingCount++ + } + } + assert.Positive(t, missingCount, "ModeEvictLRU should have evicted some early entries") +} + func TestDomainCache_Delete(t *testing.T) { c := NewDomainCache(100) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 3d3ff653979..b4301406bb7 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -99,30 +99,30 @@ type CodeCache struct { // of L1 — Get-by-ethHash bypasses addr lookup entirely. Memory cost: // duplicates code bytes vs L2 (worst case 2x byte storage); accepted // for the per-key fast-path on many-addrs-one-code workloads. - ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent - ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) + ethHashToCode *maphash.Map[[]byte] // keccak(code) → code, concurrent + ethHashCodeSize atomic.Int64 // current size in bytes (ethHash layer) // Size-only layer: ethCodeHash → int (length in bytes). Answers // EXTCODESIZE / EXTCODEHASH without loading the bytes. Geth has the // equivalent at core/state/database_code.go (1 M-entry LRU). Tiny // per-entry footprint (32B key + 8B value) so the same memory budget // gives ~1000x the hit surface vs the bytes cache. - codeSizeByEthHash *maphash.Map[int] - codeSizeEntries atomic.Int64 - codeSizeCapEntries int64 + codeSizeByEthHash *maphash.Map[int] + codeSizeEntries atomic.Int64 + codeSizeCapEntries int64 blockHash common.Hash // hash of the last block processed mu sync.RWMutex // Stats counters (atomic for concurrent access) - addrHits atomic.Uint64 - addrMisses atomic.Uint64 - codeHits atomic.Uint64 - codeMisses atomic.Uint64 - ethHashHits atomic.Uint64 - ethHashMisses atomic.Uint64 - codeSizeHits atomic.Uint64 - codeSizeMisses atomic.Uint64 + addrHits atomic.Uint64 + addrMisses atomic.Uint64 + codeHits atomic.Uint64 + codeMisses atomic.Uint64 + ethHashHits atomic.Uint64 + ethHashMisses atomic.Uint64 + codeSizeHits atomic.Uint64 + codeSizeMisses atomic.Uint64 addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index a4d0e2b40eb..af9e3d61222 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -17,36 +17,97 @@ package cache import ( + "bytes" "sync" "sync/atomic" "github.com/c2h5oh/datasize" + "github.com/elastic/go-freelru" + "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" ) -// GenericCache is a bounded concurrent cache for key-value data. +// avgBytesPerEntry is the assumption used to translate a byte budget into +// the entry-count cap that freelru.ShardedLRU is sized against. 256 B +// approximates account-record + key overhead and storage-slot value+key +// overhead in the same order of magnitude. Actual residency tracked in +// currentSize and reported via PrintStatsAndReset. +const avgBytesPerEntry = 256 + +// entry stores the full key alongside the value so callers can detect +// hash collisions (the freelru shard key is the uint64 maphash of the +// byte-string key — Go's randomized stdlib hasher, so collisions are +// rare but not impossible). size carries the byte cost of the entry so +// the OnEvict callback can update currentSize without re-running +// sizeFunc. +type entry[T any] struct { + key []byte + val T + size int +} + +// GenericCache is a sharded, LRU-evicting bounded cache for key-value +// data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - data *maphash.Map[T] + data *freelru.ShardedLRU[uint64, entry[T]] capacityB datasize.ByteSize currentSize atomic.Int64 - blockHash common.Hash - mu sync.RWMutex - hits atomic.Uint64 - misses atomic.Uint64 - inserts atomic.Uint64 - dropped atomic.Uint64 - sizeFunc func(T) int // calculates size of value in bytes -} - -// NewGenericCache creates a new GenericCache with the specified byte capacity. -func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) int) *GenericCache[T] { - return &GenericCache[T]{ - data: maphash.NewMap[T](), + mode Mode + + blockHash common.Hash + mu sync.RWMutex + + hits atomic.Uint64 + misses atomic.Uint64 + inserts atomic.Uint64 + evictions atomic.Uint64 + dropped atomic.Uint64 + + sizeFunc func(T) int +} + +func u64identity(k uint64) uint32 { return uint32(k) } + +// NewGenericCache creates a new GenericCache with the specified byte +// capacity. mode selects ModeEvictLRU (default in this tree) or ModeNoOp +// (kept for diagnostic baselines). +func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) int, mode Mode) *GenericCache[T] { + capacityEntries := uint32(uint64(capacityBytes) / avgBytesPerEntry) + if capacityEntries < 1024 { + capacityEntries = 1024 + } + if capacityEntries > 1<<22 { + capacityEntries = 1 << 22 + } + return newGenericCacheEntries(capacityBytes, capacityEntries, sizeFunc, mode) +} + +// newGenericCacheEntries builds a cache against an explicit entry-count +// cap. Used by tests that want to exercise eviction with small capacities; +// production constructs via NewGenericCache. +func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntries uint32, sizeFunc func(T) int, mode Mode) *GenericCache[T] { + if capacityEntries == 0 { + capacityEntries = 1 + } + lru, err := freelru.NewSharded[uint64, entry[T]](capacityEntries, u64identity) + if err != nil { + panic(err) + } + c := &GenericCache[T]{ + data: lru, capacityB: capacityBytes, + mode: mode, sizeFunc: sizeFunc, } + // OnEvict fires for capacity-driven LRU eviction (ModeEvictLRU) and + // for explicit Remove(). In both cases we want currentSize to follow. + lru.SetOnEvict(func(_ uint64, e entry[T]) { + c.currentSize.Add(-int64(e.size)) + c.evictions.Add(1) + }) + return c } // DomainCache wraps GenericCache[[]byte] to implement the Cache interface. @@ -54,10 +115,16 @@ type DomainCache struct { *GenericCache[[]byte] } -// NewDomainCache creates a new cache for domain data. +// NewDomainCache creates a new cache for domain data. Defaults to +// ModeEvictLRU; use NewDomainCacheMode to override. func NewDomainCache(capacityBytes datasize.ByteSize) *DomainCache { + return NewDomainCacheMode(capacityBytes, ModeEvictLRU) +} + +// NewDomainCacheMode creates a new domain cache with the given mode. +func NewDomainCacheMode(capacityBytes datasize.ByteSize, mode Mode) *DomainCache { return &DomainCache{ - GenericCache: NewGenericCache(capacityBytes, func(v []byte) int { return len(v) }), + GenericCache: NewGenericCache(capacityBytes, func(v []byte) int { return len(v) }, mode), } } @@ -82,55 +149,64 @@ func (c *DomainCache) Delete(key []byte) { // Get retrieves data for the given key. func (c *GenericCache[T]) Get(key []byte) (T, bool) { - value, ok := c.data.Get(key) - if !ok { + h := maphash.Hash(key) + e, ok := c.data.Get(h) + if !ok || !bytes.Equal(e.key, key) { c.misses.Add(1) var zero T return zero, false } c.hits.Add(1) - return value, true + return e.val, true } -// Put stores data for the given key. +// Put stores data for the given key. In ModeEvictLRU the underlying +// sharded LRU evicts cold entries when its entry-count cap is reached. +// In ModeNoOp inserts that would overflow the byte budget are dropped +// (and counted via the dropped metric). func (c *GenericCache[T]) Put(key []byte, value T) { - entrySize := int64(8 + c.sizeFunc(value)) - - // Check if key already exists - if existing, ok := c.data.Get(key); ok { - oldSize := int64(8 + c.sizeFunc(existing)) - sizeDiff := entrySize - oldSize - c.data.Set(key, value) - c.currentSize.Add(sizeDiff) + h := maphash.Hash(key) + valBytes := c.sizeFunc(value) + newSize := len(key) + valBytes + 24 + + // Existing key — update in place. Reuse the stored key buffer to + // avoid an extra allocation; the freshly-decoded value replaces the + // old one. + if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { + oldSize := existing.size + c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize}) + c.currentSize.Add(int64(newSize - oldSize)) return } - // New key. The cache currently has no eviction policy, so once full - // every subsequent new key is silently dropped until Clear/ClearWithHash/ - // ValidateAndPrepare-mismatch resets the cache. dropped vs inserts makes - // this perf cliff observable. - if c.currentSize.Load()+entrySize > int64(c.capacityB) { - c.dropped.Add(1) - return + if c.mode == ModeNoOp { + if c.currentSize.Load()+int64(newSize) > int64(c.capacityB) { + c.dropped.Add(1) + return + } } + // In ModeEvictLRU the per-shard LRU evicts the oldest entry inside + // freelru.Add when its slot cap is reached; OnEvict drops the size + // from currentSize. Eviction is per-shard, not globally-LRU — same + // trade-off code_cache.go / balcache.go / db/state/cache.go accept. - c.data.Set(key, value) - c.currentSize.Add(entrySize) + keyCopy := common.Copy(key) + c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize}) + c.currentSize.Add(int64(newSize)) c.inserts.Add(1) } // Delete removes the data for the given key. func (c *GenericCache[T]) Delete(key []byte) { - if existing, ok := c.data.Get(key); ok { - entrySize := int64(8 + c.sizeFunc(existing)) - c.data.Delete(key) - c.currentSize.Add(-entrySize) + h := maphash.Hash(key) + if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { + c.data.Remove(h) } } // Clear removes all entries from the cache. func (c *GenericCache[T]) Clear() { - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) } @@ -154,7 +230,7 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo defer c.mu.Unlock() if c.blockHash == (common.Hash{}) { - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = incomingBlockHash return true @@ -165,7 +241,7 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo return true } - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = incomingBlockHash return false @@ -175,14 +251,14 @@ func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlo func (c *GenericCache[T]) ClearWithHash(hash common.Hash) { c.mu.Lock() defer c.mu.Unlock() - c.data.Clear() + c.data.Purge() c.currentSize.Store(0) c.blockHash = hash } // Len returns the number of entries in the cache. func (c *GenericCache[T]) Len() int { - return c.data.Len() + return int(c.data.Len()) } // SizeBytes returns the current size of the cache in bytes. @@ -195,11 +271,17 @@ func (c *GenericCache[T]) CapacityBytes() datasize.ByteSize { return c.capacityB } +// Mode returns the eviction mode this cache was constructed with. +func (c *GenericCache[T]) Mode() Mode { + return c.mode +} + // PrintStatsAndReset prints cache statistics and resets counters. func (c *GenericCache[T]) PrintStatsAndReset(name string) { hits := c.hits.Swap(0) misses := c.misses.Swap(0) inserts := c.inserts.Swap(0) + evictions := c.evictions.Swap(0) dropped := c.dropped.Swap(0) total := hits + misses var hitRate float64 @@ -209,8 +291,9 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { sizeBytes := c.currentSize.Load() usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 log.Debug(name+" cache stats", + "mode", c.mode.String(), "hits", hits, "misses", misses, "hit_rate", hitRate, - "inserts", inserts, "dropped", dropped, + "inserts", inserts, "evictions", evictions, "dropped", dropped, "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) diff --git a/execution/cache/policy.go b/execution/cache/policy.go new file mode 100644 index 00000000000..7efa0fb649d --- /dev/null +++ b/execution/cache/policy.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +// Mode selects what GenericCache does when an insert would overflow the +// capacity. The seam is the same for both modes — the in-package eviction +// pop is the only behaviour difference. +// +// ModeEvictLRU is the default and matches the policy other state caches in +// the tree use (db/state/cache.go's DomainGetFromFileCache / +// IISeekInFilesCache, execution/cache/code_cache.go's addr LRUs, +// execution/balcache). +// +// ModeNoOp preserves the historical "first writers win forever" behaviour: +// once full, every new key is silently dropped (counted via the dropped +// metric). Kept as a deliberate diagnostic baseline so the regression +// bench can compare against the pre-policy behaviour without flipping +// branches. +// +// LFU is a known follow-up. Pure LRU is scan-fragile: a flood of one-shot +// keys (cold-storage bloat workloads; mainnet's long tail of single-touch +// slots) evicts the genuinely-hot working set because every cold scan is +// "more recent" than the hot entries. W-TinyLFU keeps a frequency sketch +// so single-touch entries can't displace frequently-touched ones — which +// is why reth uses it for state caches. A ModeEvictLFU would slot in +// behind the same wrapper without touching call sites. +type Mode uint8 + +const ( + ModeEvictLRU Mode = iota + ModeNoOp +) + +func (m Mode) String() string { + switch m { + case ModeEvictLRU: + return "evict" + case ModeNoOp: + return "noop" + } + return "unknown" +} diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index a9559570187..8e07f988b49 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -33,6 +33,14 @@ const ( DefaultStorageCacheBytes = 1 * datasize.GB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB + + // Per-domain avg entry size used to translate the byte budget into the + // entry-count cap the underlying sharded LRU is sized against. Account + // and storage are near-fixed: addr + record or addr+slot + value plus + // entry overhead. Commitment values are smaller branch nodes. + avgAccountEntryBytes = 96 // 20 addr + ~50 account record + 24 overhead + avgStorageEntryBytes = 88 // 52 addr+slot + ~12 value + 24 overhead + avgCommitmentEntryBytes = 80 ) // StateCache is a unified cache for domain data (Account, Storage, Code). @@ -48,13 +56,25 @@ type StateCache struct { // NewStateCache creates a new StateCache with the specified byte capacities. func NewStateCache(accountBytes, storageBytes, codeBytes, addrBytes, commitmentBytes datasize.ByteSize) *StateCache { sc := &StateCache{} - sc.caches[kv.AccountsDomain] = NewDomainCache(accountBytes) - sc.caches[kv.StorageDomain] = NewDomainCache(storageBytes) + sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, ModeEvictLRU) + sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, ModeEvictLRU) sc.caches[kv.CodeDomain] = NewCodeCache(codeBytes, addrBytes) - //sc.caches[kv.CommitmentDomain] = NewDomainCache(commitmentBytes) + //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, ModeEvictLRU) return sc } +// newDomainCacheBytes constructs a DomainCache where the entry-count cap +// is derived from the byte budget using the supplied per-domain avg. +func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode Mode) *DomainCache { + capacityEntries := uint32(uint64(capacityBytes) / uint64(avgBytes)) + if capacityEntries == 0 { + capacityEntries = 1 + } + return &DomainCache{ + GenericCache: newGenericCacheEntries(capacityBytes, capacityEntries, func(v []byte) int { return len(v) }, mode), + } +} + // NewDefaultStateCache creates a new StateCache with default byte capacities. // Account: 256MB, Storage: 512MB, Code: 512MB, Addr: 16MB func NewDefaultStateCache() *StateCache { diff --git a/execution/vm/contract.go b/execution/vm/contract.go index c6821001bb3..da7e2cb9c3b 100644 --- a/execution/vm/contract.go +++ b/execution/vm/contract.go @@ -56,7 +56,7 @@ type Contract struct { } // around 64MB cache in the worst case. -var jumpDestCache = cache.NewGenericCache[bitvec](64*datasize.MB, func(v bitvec) int { return len(v) }) +var jumpDestCache = cache.NewGenericCache[bitvec](64*datasize.MB, func(v bitvec) int { return len(v) }, cache.ModeEvictLRU) // NewContract returns a new contract environment for the execution of EVM. func NewContract(caller accounts.Address, callerAddress accounts.Address, addr accounts.Address, value uint256.Int) *Contract { From 3fec4764a22a9ff656f9f3b976833a4ce3ce9b58 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 13:42:27 +0000 Subject: [PATCH 088/120] execution/cache: STATE_CACHE_MODE env override at NewStateCache time Single env knob read once at NewStateCache. Default ModeEvictLRU, recognised override "noop" (for the regression-bench baseline so ModeEvictLRU and ModeNoOp can be compared on the same binary). Unrecognised values fall back to evict with a warn log. ModeNoOp engagement is logged at info level because the fill-and-freeze behaviour is a deliberate diagnostic state, not a production setting. Pattern matches db/state/cache.go's D_LRU_ENABLED / D_LRU knobs (dbg.EnvString from common/dbg). Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/state_cache.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 8e07f988b49..2e0d047eed6 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -18,10 +18,13 @@ package cache import ( "bytes" + "strings" "github.com/c2h5oh/datasize" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/dbg" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/execution/commitment/commitmentdb" ) @@ -54,15 +57,36 @@ type StateCache struct { } // NewStateCache creates a new StateCache with the specified byte capacities. +// Mode for the byte-budget DomainCaches (Account/Storage/Commitment) is +// read once from STATE_CACHE_MODE (evict|noop, default evict). CodeCache +// has its own LRU and is not gated by this knob. func NewStateCache(accountBytes, storageBytes, codeBytes, addrBytes, commitmentBytes datasize.ByteSize) *StateCache { + mode := stateCacheModeFromEnv() sc := &StateCache{} - sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, ModeEvictLRU) - sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, ModeEvictLRU) + sc.caches[kv.AccountsDomain] = newDomainCacheBytes(accountBytes, avgAccountEntryBytes, mode) + sc.caches[kv.StorageDomain] = newDomainCacheBytes(storageBytes, avgStorageEntryBytes, mode) sc.caches[kv.CodeDomain] = NewCodeCache(codeBytes, addrBytes) - //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, ModeEvictLRU) + //sc.caches[kv.CommitmentDomain] = newDomainCacheBytes(commitmentBytes, avgCommitmentEntryBytes, mode) return sc } +// stateCacheModeFromEnv reads STATE_CACHE_MODE once. Unset or unrecognised +// returns ModeEvictLRU. Recognised values: "evict", "noop". Logged at +// startup so operators can see the mode pick. +func stateCacheModeFromEnv() Mode { + v := strings.ToLower(strings.TrimSpace(dbg.EnvString("STATE_CACHE_MODE", ""))) + switch v { + case "", "evict": + return ModeEvictLRU + case "noop": + log.Info("[cache] STATE_CACHE_MODE=noop — Account/Storage caches will drop new keys when full (diagnostic baseline; not for production)") + return ModeNoOp + default: + log.Warn("[cache] unrecognised STATE_CACHE_MODE; defaulting to evict", "value", v) + return ModeEvictLRU + } +} + // newDomainCacheBytes constructs a DomainCache where the entry-count cap // is derived from the byte budget using the supplied per-domain avg. func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode Mode) *DomainCache { From 1a56794176aa88e907f33e83f52521f287e2e024 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 14:13:11 +0000 Subject: [PATCH 089/120] execution/cache: correct the LFU rationale in Mode docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous comment asserted "reth uses W-TinyLFU for state caches" — that is wrong on the execution hot path. Reth's cross-block state cache is `fixed-cache` (PR #21128, v1.11.0): a lock-free direct-mapped / set-associative array with collision-evict semantics. No LRU list, no LFU sketch. Their published wins (~25% newPayload p50 / +33% gas/s) came from *removing* LRU/LFU bookkeeping, not adding LFU. Where reth uses real LRU/LFU it's deliberate and not the execution cache (schnellru::LruMap for networking; moka in precompile_cache.rs explicitly configured with eviction_policy(EvictionPolicy::lru())). The docstring now reflects two follow-up policies both real: - ModeEvictFixedCache (reth's actual choice, more interesting structural option than LFU) - ModeEvictLFU (W-TinyLFU; helps mainnet steady-state, not the cycle-2 bloat fixtures which are pure cold scans) Decision criterion (per agentspecs/lfu-vs-lru-state-cache-decision-2026-05-15.md): ship ModeEvictLFU only if a 24h mainnet replay shows current sharded-LRU hit-rate < 90 % on Account/Storage. Otter is the only credible Go W-TinyLFU library; ristretto has documented correctness bugs and is disqualified for an EL hot path. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/policy.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/execution/cache/policy.go b/execution/cache/policy.go index 7efa0fb649d..6297df9108c 100644 --- a/execution/cache/policy.go +++ b/execution/cache/policy.go @@ -31,13 +31,28 @@ package cache // bench can compare against the pre-policy behaviour without flipping // branches. // -// LFU is a known follow-up. Pure LRU is scan-fragile: a flood of one-shot -// keys (cold-storage bloat workloads; mainnet's long tail of single-touch -// slots) evicts the genuinely-hot working set because every cold scan is -// "more recent" than the hot entries. W-TinyLFU keeps a frequency sketch -// so single-touch entries can't displace frequently-touched ones — which -// is why reth uses it for state caches. A ModeEvictLFU would slot in -// behind the same wrapper without touching call sites. +// Pure LRU is scan-fragile in principle: a flood of one-shot keys (mainnet's +// long tail of single-touch slots) can evict the genuinely-hot working set +// because every cold scan is "more recent" than the hot entries. Two known +// follow-up policies sit behind this seam: +// +// - ModeEvictFixedCache (reth's choice). Reth's execution cache is +// `fixed-cache`: a lock-free direct-mapped / set-associative array +// with collision-evict semantics — no LRU list, no LFU sketch. Their +// PR #21128 (v1.11.0) quoted ~25% newPayload p50 / +33% gas/s vs the +// prior moka / quick-cache attempts; the win came from *removing* +// LRU/LFU bookkeeping, not adding LFU. +// - ModeEvictLFU. W-TinyLFU (Caffeine-style) admission policy keeps a +// small frequency sketch so single-touch entries can't displace +// frequently-touched ones. Helps mainnet steady-state in principle +// (+5-15pp on Zipfian-with-scan per Caffeine literature). Does NOT +// help the cycle-2 bloat fixtures — those are pure cold scans with +// no reuse, so admission policy has nothing to admit. +// +// See agentspecs/lfu-vs-lru-state-cache-decision-2026-05-15.md for the +// full analysis (which lib to use if we ship LFU — otter, not ristretto) +// and the decision criterion (24h mainnet replay hit-rate < 90% before +// LFU is worth a new dep). type Mode uint8 const ( From 4a512ce0d3345f166992e264c3238bf303cf0ed8 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 17 May 2026 20:25:02 +0000 Subject: [PATCH 090/120] execution/cache: reduce default cache caps to 100 MB each (bench knob) Investigation knob, NOT a permanent default. Account / Storage / Code each capped at 100 MB so the bench measures layer contributions instead of being dominated by preallocated cache memory pressure (1 GB / 1 GB / 512 MB defaults push sys past the GC/page-cache pressure band on this hardware/workload mix). Permanent defaults stay at 1 GB / 1 GB / 512 MB; this commit will be reverted or dynamically gated by relative-to-available sizing. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/cache/code_cache.go | 4 ++-- execution/cache/state_cache.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index b4301406bb7..163172abda3 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -35,8 +35,8 @@ func uint64AsBytes(v *uint64) []byte { } const ( - // DefaultCodeCacheBytes is the byte limit for code cache (512 MB) - DefaultCodeCacheBytes = 512 * datasize.MB + // DefaultCodeCacheBytes is the byte limit for code cache (100 MB — investigation knob; permanent default returns to 512 MB) + DefaultCodeCacheBytes = 100 * datasize.MB // DefaultAddrCacheBytes is the byte limit for address cache (16 MB) DefaultAddrCacheBytes = 16 * datasize.MB // DefaultAddrCacheEntries derives from DefaultAddrCacheBytes assuming diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 2e0d047eed6..dbf72fb3114 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -30,10 +30,10 @@ import ( ) const ( - // DefaultAccountCacheBytes is the byte limit for account cache (1 GB) - DefaultAccountCacheBytes = 1 * datasize.GB - // DefaultStorageCacheBytes is the byte limit for storage cache (1 GB) - DefaultStorageCacheBytes = 1 * datasize.GB + // DefaultAccountCacheBytes is the byte limit for account cache (100 MB — investigation knob; permanent default returns to 1 GB) + DefaultAccountCacheBytes = 100 * datasize.MB + // DefaultStorageCacheBytes is the byte limit for storage cache (100 MB — investigation knob; permanent default returns to 1 GB) + DefaultStorageCacheBytes = 100 * datasize.MB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB From da4b091c22da0623165df886135901b6a0057e80 Mon Sep 17 00:00:00 2001 From: Mark Holt <135143369+mh0lt@users.noreply.github.com> Date: Mon, 25 May 2026 09:09:00 +0100 Subject: [PATCH 091/120] Parallel-exec correctness fixes (PR #3 of the perf stack) (#21387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR ships the parallel-exec correctness fixes from `mh/parallel-exec-fixes` onto the perf stack, packaged as a focused PR on top of [#21386 (StateCache LRU)](https://github.com/erigontech/erigon/pull/21386) which itself stacks on [#21380 (State Cache Consolidation)](https://github.com/erigontech/erigon/pull/21380). > [!IMPORTANT] > **Stacks on #21386 → #21380.** Base is `mh/perf-statecache-lru-pr`, NOT `main`. Merge order: #21380 → #21386 → this PR. > [!IMPORTANT] > **Do not merge until CI is green on both parallel and serial.** Same gating rule as #21380 / #21386. ## Scope — 13 commits from `mh/parallel-exec-fixes` Brought in via a merge commit so the bisection trail is preserved. | sha | what it fixes | |---|---| | `25053e38e9` | parallel SD-of-pre-existing-contract — the 197-line foundational fix | | `2e2bf3ccc0` | clean exit when single-block batch already covered maxBlockNum | | `6e451f5ed2` | don't emit StoragePath=0 writes from IBS.Selfdestruct | | `616a4fa0a8` | clear calc Deleted on a non-SD account write even when zero | | `d99f2f704d` | gate known parallel-exec failures behind EXEC3_PARALLEL (#21136) | | `34e83e82b7` | install per-block changeset accumulator before any of the block's writes | | `b340d7e592` | drop stale sd.mem 'Trim old version entries' comment | | `629cc23566` | O(1) CollectorWrites fee-balance update, drop dead VersionedWrites.SetBalance | | `a0ecfc7e12` | first-match-wins in CollectorWrites BalancePath index | | `445f97e446` | emit EIP-7708 Burn log under parallel-exec when coinbase self-destructs | | `5e1f5fa901` | mirror ReadAccountData SD-revival check into versionedRead | | `a5dc83f509` | drop two stale EXEC3_PARALLEL t.Skips | | `8af901104f` | drop TestReceiptHashFromRPC unit-suite RPC integration test | ## Merge conflicts resolved 3 files, 8 regions — all resolved by keeping HEAD's typed-readset / per-path revival shape and confirming HEAD already absorbs each fix's intent. See the merge commit message (`cfc4ec1418`) for the per-region rationale. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Mark Holt From 9b153620a32bdc2a595330e052d72e25b67ccbc5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 25 May 2026 21:23:28 +0000 Subject: [PATCH 092/120] execution/state: drop tautological nil checks in code-loading paths (govet) stateObject and s are both verified non-nil earlier in their respective scopes; the secondary checks at lines 749 and 783 are redundant. govet nilness check fails on these. --- execution/state/intra_block_state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/execution/state/intra_block_state.go b/execution/state/intra_block_state.go index f496473fb03..a1c6800ce3e 100644 --- a/execution/state/intra_block_state.go +++ b/execution/state/intra_block_state.go @@ -746,7 +746,7 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { if err != nil { return 0, err } - if stateObject != nil && code != nil { + if code != nil { stateObject.code = code } return len(code), nil @@ -780,7 +780,7 @@ func (sdb *IntraBlockState) GetCodeSize(addr accounts.Address) (int, error) { // effectively a hashmap probe + slice assignment. code, codeErr := sdb.stateReader.ReadAccountCode(addr) l := len(code) - if codeErr == nil && s != nil && code != nil { + if codeErr == nil && code != nil { s.code = code } err := codeErr From 6b582fd88b055f038dcabe6a6cfcf2605419efd6 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 12:43:23 +0000 Subject: [PATCH 093/120] execution/execmodule: clear BranchCache between unwindToCommonCanonical and ValidatePayload (fix 6 hive re-org failures) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ValidateChain's fork-payload path runs unwindToCommonCanonical → ValidatePayload. SD.Unwind (called transitively by unwindToCommonCanonical) does selective per-key BranchCache invalidation only for entries listed in changeset[kv.CommitmentDomain]. Commitment-trie internal branches derived during the canonical flush sit in the aggregator-scope BranchCache without appearing in any changeset entry — so the selective sweep misses them and they survive the unwind as stale canonical-lineage entries that poison the fork-payload trie reads → wrong trie root → EngineNewPayload returns INVALID on fork payloads that should be VALID. The PR author's own doctrine on HexPatriciaHashed.Reset (execution/commitment/hex_patricia_hashed.go:2970-2975) explicitly states: "The BranchCache is intentionally NOT cleared here ... Callers that need to invalidate the cache (unwind, fork validation) MUST call ClearBranchCache explicitly." SetHead unwind already follows this doctrine (set_head.go:159 calls sd.ClearBranchCache()). The ValidateChain fork-payload path didn't — this commit closes that gap. Symptom this fixes: all 6 ethereum/engine hive failures on the merge-from-main run (beeyx741q) returned INVALID where VALID expected on re-org scenarios: - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs={T,F}, Invalid P{9,10} - Re-Org Back into Canonical Chain, Depth=10, Execute Side Payload on Re-Org - Withdrawals Fork on Block N - M Block Re-Org NewPayload - Withdrawals Fork on Canonical Block N / Side Block M - 10 Block Re-Org All six are EngineNewPayload* on re-org / fork-payload scenarios — exactly the cache-pollution shape this fix addresses. Symmetric across serial and parallel jobs, confirming it's not a parallel-exec regression. Reviewer context: this is the production-side fix taratorio asked about in PR comments — the test-runner ResetBranchCache call is not "AI slop" but defense for the same gap; this commit closes the production-side gap so the test-runner reset becomes belt-and-braces rather than load-bearing. --- execution/execmodule/exec_module.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index b77676216a4..986092880ab 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -543,6 +543,20 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } + // Explicit BranchCache clear between unwind and fork-payload validation. + // SD.Unwind's selective per-key invalidation covers only entries listed in + // changeset[kv.CommitmentDomain]; commitment-trie internal branches + // derived during the canonical flush can sit in the aggregator-scope + // BranchCache without ever appearing in a changeset entry. The + // HexPatriciaHashed.Reset doctrine (execution/commitment/hex_patricia_hashed.go:2970-2975) + // explicitly requires unwind/fork-validation callers to ClearBranchCache, + // otherwise stale canonical-lineage branches poison fork-payload reads + // and produce wrong trie roots → EngineNewPayload returns INVALID on + // fork payloads that should be VALID. Observed as the 6 hive ethereum/engine + // re-org failures on this branch's merge-from-main run; the cleared + // cache lets the fork-payload trie reads see post-unwind state. + doms.ClearBranchCache() + status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError From 25eb151af6fb6604d924a88967ec9bd4b1768664 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 13:09:06 +0000 Subject: [PATCH 094/120] Revert "execution/execmodule: clear BranchCache between unwindToCommonCanonical and ValidatePayload (fix 6 hive re-org failures)" This reverts commit 6b582fd88b055f038dcabe6a6cfcf2605419efd6. --- execution/execmodule/exec_module.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 986092880ab..b77676216a4 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -543,20 +543,6 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, err } - // Explicit BranchCache clear between unwind and fork-payload validation. - // SD.Unwind's selective per-key invalidation covers only entries listed in - // changeset[kv.CommitmentDomain]; commitment-trie internal branches - // derived during the canonical flush can sit in the aggregator-scope - // BranchCache without ever appearing in a changeset entry. The - // HexPatriciaHashed.Reset doctrine (execution/commitment/hex_patricia_hashed.go:2970-2975) - // explicitly requires unwind/fork-validation callers to ClearBranchCache, - // otherwise stale canonical-lineage branches poison fork-payload reads - // and produce wrong trie roots → EngineNewPayload returns INVALID on - // fork payloads that should be VALID. Observed as the 6 hive ethereum/engine - // re-org failures on this branch's merge-from-main run; the cleared - // cache lets the fork-payload trie reads see post-unwind state. - doms.ClearBranchCache() - status, lvh, validationError, criticalError := e.forkValidator.ValidatePayload(ctx, doms, tx, header, body.RawBody(), e.logger) if criticalError != nil { return ValidationResult{}, criticalError From 0021402e1b773feea1522bb9769c00de66e48ce4 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 20:54:08 +0000 Subject: [PATCH 095/120] execution/commitment, common/maphash: txN-tagged BranchCache entries + UnwindTo watermark eviction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of replacing diffset-driven BranchCache invalidation with a self-sufficient txN-tagged cache. - branchCacheEntry gains txN uint64. Tagging conventions: sd.Flush = exact sd.txNum; snapshot reads = file endTxNum; DB latest reads = lastTxNumOfStep(step) high-water. 0 means "not tracked" — entry survives any watermark (back-compat). - Put / PutIfClean / PinEntry signatures take txN alongside step. All existing call sites pass 0 to preserve today's behavior; meaningful values land in subsequent commits as the read paths are plumbed. - New UnwindTo(maxValidTxN) evicts every entry with txN > watermark across root + pinned + LRU tail. Single watermark pass, no per-key changeset required. Pinned entries are evicted by watermark — pinning protects capacity, not correctness. - common/maphash gains Map.Range / Map.DeleteByHash and LRU.DeleteByHash to support hash-keyed iteration with in-place delete (needed because Set hashes-and-discards the byte key). Tests cover: watermark eviction across all tiers, txN=0 immortality, boundary equality (txN==watermark stays, txN==watermark+1 evicts). Step 1/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- common/maphash/maphash.go | 24 ++++ db/state/execctx/domain_shared.go | 4 +- execution/commitment/branch_cache.go | 75 +++++++++++- execution/commitment/branch_cache_test.go | 134 +++++++++++++++++++--- execution/commitment/preload.go | 2 +- execution/commitment/preload_parallel.go | 2 +- execution/commitment/trunk_pin_test.go | 10 +- 7 files changed, 221 insertions(+), 30 deletions(-) diff --git a/common/maphash/maphash.go b/common/maphash/maphash.go index 747fb45e1fd..45dfc97a00a 100644 --- a/common/maphash/maphash.go +++ b/common/maphash/maphash.go @@ -61,6 +61,24 @@ func (m *Map[V]) Clear() { m.m.Clear() } +// Range iterates over every (hash, value) pair. Iteration order is +// unspecified. Return false from fn to stop early. Concurrent +// modification during Range is permitted by the underlying xsync.Map. +// +// The original byte-key is not recoverable — Set hashes-and-discards. +// Pair with DeleteByHash to evict entries discovered via Range. +func (m *Map[V]) Range(fn func(hash uint64, v V) bool) { + m.m.Range(func(key uint64, value V) bool { + return fn(key, value) + }) +} + +// DeleteByHash removes the entry under the pre-computed hash. Use +// alongside Range when the byte-key is unknown. +func (m *Map[V]) DeleteByHash(hash uint64) { + m.m.Delete(hash) +} + // NonConcurrentMap is a non-thread-safe map that uses maphash to hash []byte keys. // Use this when you don't need concurrent access for better performance. type NonConcurrentMap[V any] struct { @@ -164,6 +182,12 @@ func (l *LRU[V]) ContainsByHash(hash uint64) bool { return l.cache.Contains(hash) } +// DeleteByHash removes the entry under the pre-computed hash. Use +// alongside Range when the byte-key is unknown. +func (l *LRU[V]) DeleteByHash(hash uint64) { + l.cache.Remove(hash) +} + // Range iterates over every (hash, value) pair without affecting LRU // recency (uses Peek under the hood). Iteration order is unspecified. // Return false from fn to stop early. diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a5ae5d0b196..265a77892cc 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,7 +780,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + sd.branchCache.Put(k, v, uint64(step), 0, "sd.Flush") }); err != nil { return err } @@ -969,7 +969,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) sd.stateCache.Put(domain, k, v) } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + sd.branchCache.Put(k, v, uint64(step), 0, "sd.GetLatest") } return v, step, nil diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9edf294d242..310a8dd4199 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -270,6 +270,18 @@ type branchCacheEntry struct { // in-memory tests but real callers should always pass the step // returned by aggTx.MeteredGetLatest / tx.GetLatest. step uint64 + + // txN is the high-water txNum the cached bytes correspond to. Used by + // UnwindTo: an unwind to txNum T evicts every entry whose txN > T. + // Tagging conventions per write source: + // - sd.Flush → sd.txNum (exact, in-memory write) + // - snapshot file → file.endTxNum (step-boundary, exact under + // the no-unwind-into-snapshots invariant) + // - DB latest read → lastTxNumOfStep(step) (conservative + // high-water within the step) + // 0 means "not tracked" — entry survives any watermark (back-compat + // with callers that haven't been migrated to pass a real txN yet). + txN uint64 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -372,7 +384,11 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. -func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { +// +// txN tags the entry with its high-water validity; UnwindTo evicts a +// pinned entry when its txN exceeds the watermark. Pinning protects +// against capacity-pressure eviction, not correctness eviction. +func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -381,6 +397,7 @@ func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin s c.pinned.Set(prefix, &branchCacheEntry{ data: dataCopy, step: step, + txN: txN, writeSeq: c.writeSeq.Add(1), origin: origin, writeTimeNanos: time.Now().UnixNano(), @@ -487,9 +504,11 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of // caller buffer lifetime. step is the on-disk file step the bytes came -// from (0 if not tracked); origin is a short label of the write site -// captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { +// from (0 if not tracked); txN is the high-water txNum the entry is +// valid through (0 if not tracked — entry survives any UnwindTo +// watermark); origin is a short label of the write site captured for +// divergence-detection diagnostics. +func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -498,6 +517,7 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string c.store(prefix, &branchCacheEntry{ data: dataCopy, step: step, + txN: txN, origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -511,11 +531,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step, txN uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, step, origin) + c.Put(prefix, data, step, txN, origin) return true } @@ -574,6 +594,49 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.tail.Delete(prefix) } +// UnwindTo evicts every cache entry whose txN > maxValidTxN across all +// tiers (root, pinned, LRU tail). Returns the number of entries evicted. +// +// This is the txN-tagged-cache invalidation primitive: a single watermark +// pass over the cache, no per-key changeset required. Entries with txN=0 +// ("not tracked") are NEVER evicted by the watermark — they're treated +// as permanently valid (callers that don't tag pay no watermark cost, +// preserving back-compat for migration). +// +// Pinned entries ARE evicted by watermark — pinning protects against +// capacity-pressure eviction, not correctness eviction. +// +// Concurrency: safe to call alongside concurrent reads. Writes during +// UnwindTo may produce a slightly larger effective working set than the +// post-call watermark suggests (a Put racing with the scan may insert an +// entry the scan already passed), but every such Put carries its own txN +// and will be caught by the next UnwindTo if the entry violates the +// invariant. +func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { + // Root tier — single slot, atomic check-and-clear. + if entry := c.root.Load(); entry != nil && entry.txN > maxValidTxN { + c.root.Store(nil) + evicted++ + } + // Pinned tier — iterate and delete in place. + c.pinned.Range(func(hash uint64, entry *branchCacheEntry) bool { + if entry != nil && entry.txN > maxValidTxN { + c.pinned.DeleteByHash(hash) + evicted++ + } + return true + }) + // LRU tail — iterate and delete in place. + c.tail.Range(func(hash uint64, entry *branchCacheEntry) bool { + if entry != nil && entry.txN > maxValidTxN { + c.tail.DeleteByHash(hash) + evicted++ + } + return true + }) + return evicted +} + // Clear empties the cache and resets stats counters across ALL tiers // (root, pinned, LRU tail). Use on Reset / fork-validation paths to // ensure stale entries from one trie root are not served against a diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 29e3534e817..e8a29b8a0f1 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -31,8 +31,8 @@ func TestBranchCache_RootPinning(t *testing.T) { rootKey := []byte{0x00} // compact-encoded empty nibble path = root branch deepKey := []byte{0x12, 0x34, 0x56} - c.Put(rootKey, []byte("root-data"), 0, "test") - c.Put(deepKey, []byte("deep-data"), 0, "test") + c.Put(rootKey, []byte("root-data"), 0, 0, "test") + c.Put(deepKey, []byte("deep-data"), 0, 0, "test") // Root reads should increment rootHits, not tailHits got, _, ok := c.Get(rootKey) @@ -55,11 +55,11 @@ func TestBranchCache_RootPinning(t *testing.T) { func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { c := NewBranchCache(10) // very small tail rootKey := []byte{0x00} - c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, "test") + c.Put(rootKey, []byte("ROOT-PERSISTS"), 0, 0, "test") // Stuff the tail well past capacity for i := 0; i < 100; i++ { - c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, "test") + c.Put([]byte{byte(i), byte(i)}, []byte{byte(i)}, 0, 0, "test") } // Root must still be there @@ -78,20 +78,20 @@ func TestBranchCache_DirtyFlag(t *testing.T) { c := NewBranchCache(100) key := []byte{0x12, 0x34} - require.True(t, c.PutIfClean(key, []byte("v1"), 0, "test")) + require.True(t, c.PutIfClean(key, []byte("v1"), 0, 0, "test")) c.MarkDirty(key) - require.False(t, c.PutIfClean(key, []byte("v2"), 0, "test"), "PutIfClean must refuse dirty entry") + require.False(t, c.PutIfClean(key, []byte("v2"), 0, 0, "test"), "PutIfClean must refuse dirty entry") got, _, _ := c.Get(key) require.Equal(t, []byte("v1"), got, "dirty entry's data preserved") // Unconditional Put replaces - c.Put(key, []byte("v3"), 0, "test") + c.Put(key, []byte("v3"), 0, 0, "test") got, _, _ = c.Get(key) require.Equal(t, []byte("v3"), got) // New entry is clean - require.True(t, c.PutIfClean(key, []byte("v4"), 0, "test")) + require.True(t, c.PutIfClean(key, []byte("v4"), 0, 0, "test")) } // TestBranchCache_GetDecoded verifies the lazy-decode read path for a @@ -106,7 +106,7 @@ func TestBranchCache_GetDecoded(t *testing.T) { require.NoError(t, err) prefix := []byte{0x12, 0x34} - c.Put(prefix, enc, 0, "test") + c.Put(prefix, enc, 0, 0, "test") // First decoded-read decodes lazily bitmap, cells, ok := c.GetDecoded(prefix) @@ -130,8 +130,8 @@ func TestBranchCache_Invalidate(t *testing.T) { c := NewBranchCache(100) rootKey := []byte{0x00} deepKey := []byte{0x12, 0x34} - c.Put(rootKey, []byte("r"), 0, "test") - c.Put(deepKey, []byte("d"), 0, "test") + c.Put(rootKey, []byte("r"), 0, 0, "test") + c.Put(deepKey, []byte("d"), 0, 0, "test") c.Invalidate(rootKey) _, _, ok := c.Get(rootKey) @@ -145,8 +145,8 @@ func TestBranchCache_Invalidate(t *testing.T) { // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("r"), 0, "test") - c.Put([]byte{0x12}, []byte("d"), 0, "test") + c.Put([]byte{0x00}, []byte("r"), 0, 0, "test") + c.Put([]byte{0x12}, []byte("d"), 0, 0, "test") _, _, _ = c.Get([]byte{0x00}) _, _, _ = c.Get([]byte{0x12}) @@ -166,8 +166,8 @@ func TestBranchCache_Clear(t *testing.T) { // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) - c.Put([]byte{0x00}, []byte("rrr"), 0, "test") - c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, "test") + c.Put([]byte{0x00}, []byte("rrr"), 0, 0, "test") + c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, 0, "test") _, _, _ = c.Get([]byte{0x00}) _, _, _ = c.Get([]byte{0x12, 0x34}) _, _, _ = c.Get([]byte{0xff}) // tail miss @@ -185,3 +185,107 @@ func TestBranchCache_Stats(t *testing.T) { // Sanity: format doesn't blow up if we read it require.True(t, strings.HasPrefix(s, "branch-cache ")) } + +// TestBranchCache_UnwindTo_EvictsByTxNWatermark verifies that UnwindTo +// evicts every entry whose txN > watermark across root + pinned + tail +// tiers, and that entries with txN <= watermark survive untouched. +func TestBranchCache_UnwindTo_EvictsByTxNWatermark(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKeyKeep := []byte{0xa0, 0xb0} + tailKeyEvict := []byte{0xa0, 0xb1} + + // txN=50 entries in every tier — these should survive a watermark of 60. + c.Put(rootKey, []byte("root-keep"), 0, 50, "test") + c.PinEntry(pinnedKey, []byte("pinned-keep"), 0, 50, "test") + c.Put(tailKeyKeep, []byte("tail-keep"), 0, 50, "test") + + // txN=100 tail entry — should be evicted at watermark=60. + c.Put(tailKeyEvict, []byte("tail-evict"), 0, 100, "test") + + evicted := c.UnwindTo(60) + require.Equal(t, 1, evicted, "only the txN=100 tail entry should be evicted") + + _, _, ok := c.Get(rootKey) + require.True(t, ok, "root entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(pinnedKey) + require.True(t, ok, "pinned entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(tailKeyKeep) + require.True(t, ok, "tail entry with txN=50 must survive watermark=60") + _, _, ok = c.Get(tailKeyEvict) + require.False(t, ok, "tail entry with txN=100 must be evicted by watermark=60") +} + +// TestBranchCache_UnwindTo_EvictsAcrossAllTiers verifies eviction reaches +// the root + pinned tiers, not just the LRU tail. +func TestBranchCache_UnwindTo_EvictsAcrossAllTiers(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKey := []byte{0xa0, 0xb0} + + c.Put(rootKey, []byte("root"), 0, 100, "test") + c.PinEntry(pinnedKey, []byte("pinned"), 0, 100, "test") + c.Put(tailKey, []byte("tail"), 0, 100, "test") + + evicted := c.UnwindTo(50) + require.Equal(t, 3, evicted, "every entry with txN > watermark must be evicted") + + _, _, ok := c.Get(rootKey) + require.False(t, ok, "root entry must be evicted") + _, _, ok = c.Get(pinnedKey) + require.False(t, ok, "pinned entry must be evicted (pinning protects capacity, not correctness)") + _, _, ok = c.Get(tailKey) + require.False(t, ok, "tail entry must be evicted") +} + +// TestBranchCache_UnwindTo_TxNZeroIsImmortal verifies that entries +// tagged with txN=0 ("not tracked") are NEVER evicted by any watermark. +// Preserves back-compat with callers that haven't been migrated to +// pass real txN values. +func TestBranchCache_UnwindTo_TxNZeroIsImmortal(t *testing.T) { + c := NewBranchCache(100) + + rootKey := []byte{0x00} + pinnedKey := []byte{0x12, 0x34, 0x56} + tailKey := []byte{0xa0, 0xb0} + + c.Put(rootKey, []byte("root"), 0, 0, "test") + c.PinEntry(pinnedKey, []byte("pinned"), 0, 0, "test") + c.Put(tailKey, []byte("tail"), 0, 0, "test") + + evicted := c.UnwindTo(0) + require.Equal(t, 0, evicted, "no entry with txN=0 should be evicted") + evicted = c.UnwindTo(1_000_000) + require.Equal(t, 0, evicted, "no entry with txN=0 should be evicted even at huge watermark") + + _, _, ok := c.Get(rootKey) + require.True(t, ok, "txN=0 root entry must survive any watermark") + _, _, ok = c.Get(pinnedKey) + require.True(t, ok, "txN=0 pinned entry must survive any watermark") + _, _, ok = c.Get(tailKey) + require.True(t, ok, "txN=0 tail entry must survive any watermark") +} + +// TestBranchCache_UnwindTo_BoundaryEqualsWatermark verifies the +// inequality at the watermark itself: entries at exactly txN==watermark +// are NOT evicted (the rule is txN > watermark). +func TestBranchCache_UnwindTo_BoundaryEqualsWatermark(t *testing.T) { + c := NewBranchCache(100) + + atKey := []byte{0xa0, 0xb0} + aboveKey := []byte{0xa0, 0xb1} + c.Put(atKey, []byte("at"), 0, 100, "test") + c.Put(aboveKey, []byte("above"), 0, 101, "test") + + evicted := c.UnwindTo(100) + require.Equal(t, 1, evicted, "only the txN=101 entry must be evicted; txN=100 stays") + + _, _, ok := c.Get(atKey) + require.True(t, ok, "entry at txN==watermark must survive") + _, _, ok = c.Get(aboveKey) + require.False(t, ok, "entry at txN>watermark must be evicted") +} diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..d578d43d914 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -124,7 +124,7 @@ func (p *ContractTrunkPreload) Run( break } - cache.PinEntry(prefix, v, step, "preload-trunk") + cache.PinEntry(prefix, v, step, 0, "preload-trunk") // Copy prefix because nibbles.HexToCompact may return a slice // over a reused buffer; we need a stable handle for later // Invalidate on demotion. diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 6f8374a0d32..1e8ff8a464f 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -164,7 +164,7 @@ func (p *ContractTrunkPreloadParallel) Run( budgetHit = true return false } - cache.PinEntry(pk.key, v, 0, "preload-trunk-parallel-resumable") + cache.PinEntry(pk.key, v, 0, 0, "preload-trunk-parallel-resumable") kc := make([]byte, len(pk.key)) copy(kc, pk.key) p.pinnedPrefixes = append(p.pinnedPrefixes, kc) diff --git a/execution/commitment/trunk_pin_test.go b/execution/commitment/trunk_pin_test.go index e1c58b2dc35..a8d114c2fed 100644 --- a/execution/commitment/trunk_pin_test.go +++ b/execution/commitment/trunk_pin_test.go @@ -41,7 +41,7 @@ func fakeReader(saturated bool) CommitmentReader { func TestPinEntry_AndPinnedCount(t *testing.T) { c := NewBranchCache(8) require.Equal(t, 0, c.PinnedCount()) - c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, "test") + c.PinEntry([]byte{0x12, 0x34, 0x56}, []byte{0xab, 0xcd}, 1, 0, "test") require.Equal(t, 1, c.PinnedCount()) // Pinned entry is hit, not the tail. @@ -56,7 +56,7 @@ func TestPinEntry_AndPinnedCount(t *testing.T) { // deletes a pinned entry, not just LRU/root. func TestInvalidate_RemovesFromPinned(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0x99}, 1, 0, "test") require.Equal(t, 1, c.PinnedCount()) c.Invalidate([]byte{0x12, 0x34}) @@ -69,7 +69,7 @@ func TestInvalidate_RemovesFromPinned(t *testing.T) { // pinned entries (was a correctness gap pre-fix). func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, "preload") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 5, 0, "preload") data, origin, _, _, ok := c.GetWithOrigin([]byte{0x12, 0x34}) require.True(t, ok) require.Equal(t, []byte{0xab}, data) @@ -80,7 +80,7 @@ func TestGetWithOrigin_ChecksPinnedTier(t *testing.T) { // pinned-stats counters (was a correctness gap pre-fix). func TestClear_ResetsPinnedTier(t *testing.T) { c := NewBranchCache(8) - c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xab}, 1, 0, "test") _, _, _ = c.Get([]byte{0x12, 0x34}) // bump pinned hit c.Clear() require.Equal(t, 0, c.PinnedCount()) @@ -99,7 +99,7 @@ func TestMissCallback_FiresOnTripleMiss(t *testing.T) { }) // Hit (pinned): callback should NOT fire. - c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, "test") + c.PinEntry([]byte{0x12, 0x34}, []byte{0xff}, 1, 0, "test") _, _, _ = c.Get([]byte{0x12, 0x34}) require.Equal(t, uint64(0), fired.Load(), "pinned hit must not fire callback") From f972499e1e502e0eb77e38b0155d80c8f25f128f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 21:11:50 +0000 Subject: [PATCH 096/120] db/state, db/state/execctx: plumb real txN to BranchCache writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 2+3 of the txN-tagged BranchCache rollout. Now every write to the cache carries a meaningful txN so UnwindTo (added in Step 1) can actually evict the right entries on unwind. - AggregatorRoTx gains MeteredGetLatestWithTxN + internal getLatestWithTxN for CommitmentDomain. DB-sourced reads tag with lastTxNumOfStep(step) (conservative high-water within the step — exact write txN isn't recoverable from the step-keyed DB record). File-sourced reads tag with fileEndTxNum (step-boundary exact; unwind can't cross into snapshots, so always <= any legal watermark). - sd.GetLatest type-switches on tx.AggTx() to prefer MeteredGetterWithTxN, falling back to legacy MeteredGetter (passes txN=0, "not tracked") and finally tx.GetLatest. The captured txN flows to branchCache.Put. - sd.Flush callback tags entries with sd.txNum (the frontier at flush time). UnwindTo can evict cache entries from any unwound batch via a single watermark pass. - Non-CommitmentDomain reads return txN=0 — only CommitmentDomain participates in the txN-tagged BranchCache today. Step 2+3/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- db/state/aggregator.go | 50 ++++++++++++++++++++++++------- db/state/execctx/domain_shared.go | 28 ++++++++++++++--- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 387e4f4a906..68f49eaf639 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2482,36 +2482,66 @@ func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, return at.getLatest(domain, k, tx, maxStep, metrics, start) } +// MeteredGetLatestWithTxN is the txN-aware variant of MeteredGetLatest. +// In addition to (value, step), it returns the high-water txNum the +// returned value corresponds to — used to tag BranchCache entries so +// UnwindTo can evict by watermark. For non-CommitmentDomain reads, +// txN=0 ("not tracked"); only CommitmentDomain participates in the +// txN-tagged BranchCache today. +// +// Tagging conventions for CommitmentDomain: +// - DB-sourced: txN = lastTxNumOfStep(step) (conservative high-water +// within the step — exact write txN isn't recoverable from the +// step-keyed DB record). +// - File-sourced: txN = fileEndTxNum (exact; can't unwind into +// snapshots, so the high-water is safe by invariant). +func (at *AggregatorRoTx) MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { + return at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) +} + func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { + v, step, _, ok, err = at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) + return v, step, ok, err +} + +// getLatestWithTxN is the txN-aware variant of getLatest. It mirrors +// the source split (DB vs. files) and returns the high-water txNum the +// returned value corresponds to. See MeteredGetLatestWithTxN for the +// tagging convention. +func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { if domain != kv.CommitmentDomain { - return at.d[domain].getLatest(k, tx, maxStep, metrics, start) + // Non-Commitment domains don't participate in the txN-tagged + // BranchCache; return txN=0 ("not tracked"). The cache layer + // treats this as immortal-by-watermark. + v, step, ok, err = at.d[domain].getLatest(k, tx, maxStep, metrics, start) + return v, step, 0, ok, err } v, step, ok, err = at.d[domain].getLatestFromDb(k, tx) if err != nil { - return nil, kv.Step(0), false, err + return nil, kv.Step(0), 0, false, err } if ok && step <= maxStep { if metrics != nil && dbg.KVReadLevelledMetrics { metrics.UpdateDbReads(domain, start) } - return v, step, true, nil + // DB-sourced: high-water within the step (the actual write + // txN isn't recoverable from the step-keyed record). + return v, step, lastTxNumOfStep(step, at.StepSize()), true, nil } v, found, fileStartTxNum, fileEndTxNum, err := at.d[domain].getLatestFromFiles(k, 0) if !found { - return nil, kv.Step(0), false, err + return nil, kv.Step(0), 0, false, err } if metrics != nil && dbg.KVReadLevelledMetrics { - // UpdateFileReadsUnique tracks both total reads and distinct - // prefixes. The ratio FileReadCount / UniqueFileReadCount is - // the read amplification factor — useful when investigating - // whether the same prefixes are being re-read from file layer - // (cache misses on hot prefixes). metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) - return v, kv.Step(fileEndTxNum / at.StepSize()), found, err + // File-sourced: tag with fileEndTxNum (step boundary, exact). + // Snapshots are immutable and unwind can't cross into them, so + // fileEndTxNum is always <= any legal unwind watermark. + return v, kv.Step(fileEndTxNum / at.StepSize()), fileEndTxNum, found, err } func (at *AggregatorRoTx) DebugGetLatestFromDB(domain kv.Domain, key []byte, tx kv.Tx) ([]byte, kv.Step, bool, error) { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 265a77892cc..f0bb1506ac6 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,7 +780,11 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - sd.branchCache.Put(k, v, uint64(step), 0, "sd.Flush") + // sd.txNum is the frontier at flush time; all entries in + // this batch were written at or before it. Tagging with + // sd.txNum lets UnwindTo evict cache entries from any + // unwound batch via a single watermark pass. + sd.branchCache.Put(k, v, uint64(step), sd.txNum, "sd.Flush") }); err != nil { return err } @@ -912,6 +916,15 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) type MeteredGetter interface { MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } + // MeteredGetterWithTxN is the txN-aware sibling. Used to tag + // CommitmentDomain BranchCache puts so UnwindTo can evict by + // watermark. AggregatorRoTx implements both; the cache-relevant + // read path prefers this one and falls back to MeteredGetter + // (passing txN=0, "not tracked") when only the legacy interface + // is available — e.g. test stubs. + type MeteredGetterWithTxN interface { + MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) + } // stateCache holds in-flight values from previous transactions in the same batch // that haven't been flushed to DB yet. Early return keeps correctness AND performance. @@ -955,9 +968,16 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } - if aggTx, ok := tx.AggTx().(MeteredGetter); ok { + // Capture txN from the read for BranchCache tagging. Prefer the + // txN-aware variant; fall back to the legacy method (txN stays 0, + // "not tracked" — cache entry survives any UnwindTo watermark). + var readTxN uint64 + switch aggTx := tx.AggTx().(type) { + case MeteredGetterWithTxN: + v, step, readTxN, _, err = aggTx.MeteredGetLatestWithTxN(domain, k, tx, maxStep, &sd.metrics, start) + case MeteredGetter: v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) - } else { + default: v, step, err = tx.GetLatest(domain, k) } if err != nil { @@ -969,7 +989,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) sd.stateCache.Put(domain, k, v) } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), 0, "sd.GetLatest") + sd.branchCache.Put(k, v, uint64(step), readTxN, "sd.GetLatest") } return v, step, nil From b93969affe95ba362dfca5cc75a298176e714d47 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 21:18:36 +0000 Subject: [PATCH 097/120] db/state/execctx, execution/commitment: wire BranchCache.UnwindTo into SD.Unwind, retire the doctrine comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SharedDomains.Unwind now invalidates the BranchCache via a single UnwindTo(txNumUnwindTo) call — every entry whose txN > watermark is evicted across root + pinned + LRU tail in one pass. This replaces the changeset[CommitmentDomain]-walking invalidation that was load-bearing for cache correctness but silently broken at every write site that lacked an active SetChangesetAccumulator (frozen-block commits, beyond-reorg commits, statetest paths). The txN-tagged cache is correct by construction regardless of where the writes originate. HexPatriciaHashed.Reset's doctrine comment is updated: the cache no longer requires explicit ClearBranchCache calls on unwind/fork- validation paths — SharedDomains.Unwind's UnwindTo handles it via the txN-tagged eviction. Step 4+5/n of #21380 chosen-fix design — see pr-21380-txn-tagged-cache-design-2026-05-26 memory pin. --- db/state/execctx/domain_shared.go | 20 +++++++++++--------- execution/commitment/hex_patricia_hashed.go | 5 +++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index f0bb1506ac6..5aa0cb4beb5 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -568,15 +568,17 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // After unwind the canonical commitment-domain values are rolled - // back; any cached entries for keys touched in the unwind window - // now hold stale bytes vs the post-unwind canonical state. Drop - // those specifically — the rest of the cache (including pinned - // branches whose keys weren't in the changeset) stays warm. - if sd.branchCache != nil && changeset != nil { - for _, diff := range changeset[kv.CommitmentDomain] { - sd.branchCache.Invalidate([]byte(diff.Key)) - } + // Drop every BranchCache entry whose txN > txNumUnwindTo via a + // single watermark pass. Self-sufficient invalidation — no + // dependency on changeset[CommitmentDomain] being correctly + // populated by upstream writers. The diffset-driven invalidation + // this replaced was load-bearing for cache correctness but silently + // broken at every write site that lacked an active + // SetChangesetAccumulator (frozen-block commits, beyond-reorg + // commits, statetest paths). The txN-tagged cache is correct by + // construction regardless of where the writes originate. + if sd.branchCache != nil { + sd.branchCache.UnwindTo(txNumUnwindTo) } } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 060057af560..7de86533cf7 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2971,8 +2971,9 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { // // The BranchCache is intentionally NOT cleared here: with aggregator-scope // lifetime the cache must survive between commitment calculations to deliver -// cross-block hits. Callers that need to invalidate the cache (unwind, fork -// validation) MUST call ClearBranchCache explicitly. +// cross-block hits. Unwind correctness is handled by the cache's own +// txN-tagged eviction in SharedDomains.Unwind — no explicit +// invalidation is required on the commitment path. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false From c9a3deb6a305d323e79d059415246f49ed320225 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 26 May 2026 23:48:08 +0000 Subject: [PATCH 098/120] execution/commitment, db/state, execmodule: trim comments per project policy + drop redundant SetHead cache clear MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cleanups: 1. SharedDomains.Unwind invokes branchCache.UnwindTo (added earlier in this PR), which evicts entries by txN watermark. The explicit sd.ClearBranchCache() in ExecModule.SetHead is now redundant — the unwind path already invalidates the right entries, and KeyCommitmentState is filtered out of the cache at Put time so it can't accumulate stale entries. Removed. With this change the BranchCache is never wholesale-cleared in production code paths; all invalidation flows through UnwindTo's watermark eviction. 2. Trim verbose comments per project policy ("default to writing no comment", keep WHY-only, drop restatements / task references / over-explanations of standard Go). Net -430 lines across 14 files, mostly multi-line doc blocks that restated what the code does or narrated PR context. No code logic changed; the WHY-comments that capture non-obvious invariants, workarounds, or surprising edge cases are preserved. --- db/state/aggregator.go | 34 +--- db/state/execctx/domain_shared.go | 25 +-- execution/commitment/adaptive_pin.go | 166 +++--------------- execution/commitment/branch_cache.go | 54 ++---- execution/commitment/branch_decode.go | 41 ++--- execution/commitment/commitment.go | 53 ++---- .../commitmentdb/commitment_context.go | 50 ++---- execution/commitment/hex_patricia_hashed.go | 8 +- execution/commitment/preload.go | 91 +++------- execution/commitment/preload_parallel.go | 162 ++++------------- execution/commitment/preload_ranges.go | 9 +- execution/commitment/trunk_pin_metrics.go | 21 +-- execution/commitment/warmuper.go | 37 +--- execution/execmodule/set_head.go | 9 - 14 files changed, 165 insertions(+), 595 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 68f49eaf639..9d8bc40b0ae 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2482,19 +2482,9 @@ func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, return at.getLatest(domain, k, tx, maxStep, metrics, start) } -// MeteredGetLatestWithTxN is the txN-aware variant of MeteredGetLatest. -// In addition to (value, step), it returns the high-water txNum the -// returned value corresponds to — used to tag BranchCache entries so -// UnwindTo can evict by watermark. For non-CommitmentDomain reads, -// txN=0 ("not tracked"); only CommitmentDomain participates in the -// txN-tagged BranchCache today. -// -// Tagging conventions for CommitmentDomain: -// - DB-sourced: txN = lastTxNumOfStep(step) (conservative high-water -// within the step — exact write txN isn't recoverable from the -// step-keyed DB record). -// - File-sourced: txN = fileEndTxNum (exact; can't unwind into -// snapshots, so the high-water is safe by invariant). +// MeteredGetLatestWithTxN returns the high-water txN alongside (value, +// step) for tagging BranchCache entries so UnwindTo can evict by +// watermark. Non-CommitmentDomain reads return txN=0. func (at *AggregatorRoTx) MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { return at.getLatestWithTxN(domain, k, tx, maxStep, metrics, start) } @@ -2504,15 +2494,8 @@ func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxSte return v, step, ok, err } -// getLatestWithTxN is the txN-aware variant of getLatest. It mirrors -// the source split (DB vs. files) and returns the high-water txNum the -// returned value corresponds to. See MeteredGetLatestWithTxN for the -// tagging convention. func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) { if domain != kv.CommitmentDomain { - // Non-Commitment domains don't participate in the txN-tagged - // BranchCache; return txN=0 ("not tracked"). The cache layer - // treats this as immortal-by-watermark. v, step, ok, err = at.d[domain].getLatest(k, tx, maxStep, metrics, start) return v, step, 0, ok, err } @@ -2525,8 +2508,8 @@ func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, if metrics != nil && dbg.KVReadLevelledMetrics { metrics.UpdateDbReads(domain, start) } - // DB-sourced: high-water within the step (the actual write - // txN isn't recoverable from the step-keyed record). + // DB-sourced: tag with the step's high-water; the exact write + // txN isn't recoverable from the step-keyed record. return v, step, lastTxNumOfStep(step, at.StepSize()), true, nil } @@ -2535,12 +2518,13 @@ func (at *AggregatorRoTx) getLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, return nil, kv.Step(0), 0, false, err } if metrics != nil && dbg.KVReadLevelledMetrics { + // UpdateFileReadsUnique tracks total + distinct prefixes; the + // ratio is the read amplification factor (hot prefixes re-read). metrics.UpdateFileReadsUnique(domain, k, start) } v, err = at.replaceShortenedKeysInBranch(k, commitment.BranchData(v), fileStartTxNum, fileEndTxNum) - // File-sourced: tag with fileEndTxNum (step boundary, exact). - // Snapshots are immutable and unwind can't cross into them, so - // fileEndTxNum is always <= any legal unwind watermark. + // File-sourced: tag with fileEndTxNum; snapshots are immutable and + // unwind can't cross them, so this is always <= any legal watermark. return v, kv.Step(fileEndTxNum / at.StepSize()), fileEndTxNum, found, err } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 5aa0cb4beb5..bc7e356106b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -568,15 +568,6 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // Drop every BranchCache entry whose txN > txNumUnwindTo via a - // single watermark pass. Self-sufficient invalidation — no - // dependency on changeset[CommitmentDomain] being correctly - // populated by upstream writers. The diffset-driven invalidation - // this replaced was load-bearing for cache correctness but silently - // broken at every write site that lacked an active - // SetChangesetAccumulator (frozen-block commits, beyond-reorg - // commits, statetest paths). The txN-tagged cache is correct by - // construction regardless of where the writes originate. if sd.branchCache != nil { sd.branchCache.UnwindTo(txNumUnwindTo) } @@ -782,10 +773,6 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.branchCache.Invalidate(k) return } - // sd.txNum is the frontier at flush time; all entries in - // this batch were written at or before it. Tagging with - // sd.txNum lets UnwindTo evict cache entries from any - // unwound batch via a single watermark pass. sd.branchCache.Put(k, v, uint64(step), sd.txNum, "sd.Flush") }); err != nil { return err @@ -918,12 +905,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) type MeteredGetter interface { MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } - // MeteredGetterWithTxN is the txN-aware sibling. Used to tag - // CommitmentDomain BranchCache puts so UnwindTo can evict by - // watermark. AggregatorRoTx implements both; the cache-relevant - // read path prefers this one and falls back to MeteredGetter - // (passing txN=0, "not tracked") when only the legacy interface - // is available — e.g. test stubs. + // MeteredGetterWithTxN exposes the txN of the read so the + // BranchCache entry can be tagged; falls back to MeteredGetter + // when only the legacy interface is implemented (test stubs). type MeteredGetterWithTxN interface { MeteredGetLatestWithTxN(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, txN uint64, ok bool, err error) } @@ -970,9 +954,6 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } - // Capture txN from the read for BranchCache tagging. Prefer the - // txN-aware variant; fall back to the legacy method (txN stays 0, - // "not tracked" — cache entry survives any UnwindTo watermark). var readTxN uint64 switch aggTx := tx.AggTx().(type) { case MeteredGetterWithTxN: diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index c3668e01e7e..baff8961978 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -18,44 +18,17 @@ import ( ) // AdaptivePinControllerConfig sets the policy knobs for the adaptive -// trunk-pin controller. Defaults are tuned for the SSTORE-bloat -// workload class (single contract dominating storage reads); other -// workload classes may want different values. +// trunk-pin controller. Defaults target the SSTORE-bloat workload class +// (single contract dominating storage reads). type AdaptivePinControllerConfig struct { - // PromoteThresholdMisses — minimum cache misses (per block) for a - // contract to be promoted. Default 100. - PromoteThresholdMisses uint64 - - // MaxPromotedContracts — hard cap on simultaneously-pinned - // contracts. Bounds total pin RAM = MaxPromotedContracts × - // PerContractMaxBudgetBytes. Default 8 (× 64 MiB = 512 MiB max). - MaxPromotedContracts int - - // DemoteCooldownBlocks — number of consecutive blocks with zero - // misses for a promoted contract before it gets demoted (and its - // pin set invalidated). Default 5. - DemoteCooldownBlocks int - - // InitialViewBudgetBytes — RAM budget for the synchronous - // initial-view preload at promotion. Default 4 MiB (covers - // d=64-67 with headroom; ~1.4 s on cold disk). - InitialViewBudgetBytes int - - // ExtensionBudgetBytes — RAM budget for the per-block extension - // step on already-promoted contracts. Default 8 MiB (~25 K - // branches per block at typical entry cost). - ExtensionBudgetBytes int - - // PerContractMaxBudgetBytes — hard ceiling on the cumulative pin - // budget per contract. Extensions stop when this is reached even - // if the BFS queue isn't empty. Default 64 MiB (matches the - // per-contract max enforced in the env hook). + PromoteThresholdMisses uint64 + MaxPromotedContracts int + DemoteCooldownBlocks int + InitialViewBudgetBytes int + ExtensionBudgetBytes int PerContractMaxBudgetBytes int } -// DefaultAdaptivePinControllerConfig returns the production defaults. -// Conservative across the board so default-on shipping doesn't pin -// memory speculatively. func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { return AdaptivePinControllerConfig{ PromoteThresholdMisses: 100, @@ -69,52 +42,31 @@ func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { // AdaptivePinController watches per-contract miss pressure on a // BranchCache and decides which contracts to pin (with a sync initial -// view) and grow (per-block extension) or demote (invalidate the pin +// view), grow (per-block extension), or demote (invalidate the pin // set after sustained inactivity). -// -// Lifecycle: -// - Construct with NewAdaptivePinController, wire to a cache via Bind -// - On every triple-miss, the cache calls back; the controller -// records a per-contract miss count -// - The host (SD or stage loop) calls OnBlockComplete at block -// boundaries with a CommitmentReader factory; the controller -// then promotes / extends / demotes based on the per-block -// miss snapshot -// - Optional: host installs a parallel-mode resolver factory via -// SetParallelMode so promote/extend uses the wave-BFS parallel -// preload (file-only batch resolver + MDBX-overlay dbBranches) -// instead of the serial CommitmentReader BFS. type AdaptivePinController struct { cache *BranchCache cfg AdaptivePinControllerConfig logger log.Logger - // misses holds per-contract miss counts since last OnBlockComplete. - // sync.Map is appropriate: writes are frequent (every triple-miss), - // reads happen once per block boundary. misses sync.Map // [32]byte → *atomic.Uint64 - // states holds per-promoted-contract bookkeeping. Protected by mu. mu sync.Mutex states map[[32]byte]*adaptiveContractState - // Parallel-mode plumbing. nil = serial-BFS path (CommitmentReader). - // Set together via SetParallelMode; both required for parallel mode. parallelResolverFactory ParallelResolverFactory dbBranchesProvider DbBranchesProvider } // ParallelResolverFactory builds a fresh BatchBranchResolver for one // OnBlockComplete call. release() is invoked after the controller is done -// with the resolver (e.g. to tear down per-call worker txs). The factory -// may return (nil, nil, err) to indicate the controller should fall back -// to the serial-BFS path for this block. +// with the resolver. Returning (nil, nil, err) makes the controller fall +// back to the serial-BFS path for this block. type ParallelResolverFactory func() (resolve BatchBranchResolver, release func(), err error) // DbBranchesProvider returns the MDBX-resident branch overlay for one // contract — values shadow file values in the parallel preload's wave. -// Empty/nil result is valid (means "no MDBX overlay; resolver is -// authoritative"); the controller treats it as a non-error. +// Empty/nil result is valid (no overlay; resolver is authoritative). type DbBranchesProvider func(contractHash []byte) map[string][]byte type adaptiveContractState struct { @@ -125,7 +77,6 @@ type adaptiveContractState struct { coldBlocksInARow int } -// pinnedTotal returns the cumulative pinned-count regardless of preload mode. func (s *adaptiveContractState) pinnedTotal() int { if s.parallel != nil { return s.parallel.PinnedTotal() @@ -133,7 +84,6 @@ func (s *adaptiveContractState) pinnedTotal() int { return s.preload.PinnedTotal() } -// usedBytes returns the cumulative used-bytes regardless of preload mode. func (s *adaptiveContractState) usedBytes() int { if s.parallel != nil { return s.parallel.UsedBytes() @@ -141,7 +91,6 @@ func (s *adaptiveContractState) usedBytes() int { return s.preload.UsedBytes() } -// queueRemaining returns the un-processed queue size regardless of mode. func (s *adaptiveContractState) queueRemaining() int { if s.parallel != nil { return s.parallel.QueueRemaining() @@ -149,7 +98,6 @@ func (s *adaptiveContractState) queueRemaining() int { return s.preload.QueueRemaining() } -// pinnedPrefixes returns the pin set for cache invalidation on demote. func (s *adaptiveContractState) pinnedPrefixes() [][]byte { if s.parallel != nil { return s.parallel.PinnedPrefixes() @@ -157,10 +105,6 @@ func (s *adaptiveContractState) pinnedPrefixes() [][]byte { return s.preload.PinnedPrefixes() } -// NewAdaptivePinController constructs a controller bound to the given -// cache. Use Bind to install the cache miss-callback (separate so a -// caller can keep a controller for telemetry without wiring it into -// the read path). func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfig, logger log.Logger) *AdaptivePinController { if cfg.InitialViewBudgetBytes <= 0 { cfg.InitialViewBudgetBytes = 4 << 20 @@ -188,31 +132,16 @@ func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfi } } -// Bind installs the controller's miss-callback on the cache. After -// Bind, every triple-miss attributable to a storage-trunk prefix -// (length >= 33 B) is counted toward the corresponding contract. -// -// Safe to call multiple times — Bind replaces any prior callback. -// Call SetMissCallback(nil) on the cache directly to unbind. +// Bind installs the controller's miss-callback on the cache. +// Safe to call multiple times — replaces any prior callback. func (c *AdaptivePinController) Bind() { c.cache.SetMissCallback(c.onCacheMiss) } -// SetParallelMode switches the controller to the wave-BFS parallel preload -// (ContractTrunkPreloadParallel) for both initial-view and per-block -// extension. The factory builds a tx-scoped resolver per OnBlockComplete -// (file-only); the provider returns the MDBX-resident branch overlay per -// contract so the freshest values shadow file values. -// -// Either argument may be nil to clear it. When factory==nil the controller -// uses the serial-BFS CommitmentReader path (existing default behavior). -// When factory is set but the factory call returns nil resolver/err, the -// controller falls back to serial-BFS for that block — the saved parallel -// state survives the block and resumes on the next successful factory call. -// -// Must be called before the first OnBlockComplete; calling it later is -// allowed but changes behavior at the next block boundary only — already- -// promoted contracts keep their existing serial/parallel state. +// SetParallelMode switches promote/extend to the wave-BFS parallel preload. +// Either argument may be nil to clear; with factory==nil the controller uses +// the serial-BFS CommitmentReader path. Already-promoted contracts keep +// their existing serial/parallel state until next demote. func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { c.mu.Lock() defer c.mu.Unlock() @@ -230,28 +159,15 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { } // OnBlockComplete consumes the per-block miss snapshot and decides -// promotions, extensions, and demotions. Called by the host at block -// boundaries (after SD.Flush). The reader is the CommitmentReader -// for the just-committed state, used by initial-view preload and -// per-block extension. -// -// Synchronous: initial-view preloads run inline so the new pin set -// is available for the NEXT block's reads. Extensions also run -// inline; sized so per-block work fits within typical inter-block -// idle (~5 s of preload work for ExtensionBudgetBytes=8 MiB). -// -// Logs a [adaptive-pin] line per block when any state changes or -// promoted contracts exist. +// promotions, extensions, and demotions. Synchronous — preloads run +// inline so the new pin set is available for the next block's reads. func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { misses := c.snapshotMisses() c.mu.Lock() defer c.mu.Unlock() - // Build the per-call parallel resolver up-front (one factory call per - // OnBlockComplete shared across all contracts this block). nil when - // parallel mode isn't installed, or the factory failed (we fall back to - // the serial-BFS reader path). + // One factory call per block, shared across all contracts. nil falls back to serial. var parallelResolve BatchBranchResolver var releaseParallel func() if c.parallelResolverFactory != nil { @@ -269,13 +185,11 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui var promoted, extended, demoted int - // Already-promoted contracts: extend on hot, demote on cold. for hash, state := range c.states { n, hadMisses := misses[hash] if hadMisses && n > 0 { state.coldBlocksInARow = 0 delete(misses, hash) - // Extend if budget remains and queue not empty. if state.queueRemaining() > 0 && state.usedBytes() < c.cfg.PerContractMaxBudgetBytes { remaining := c.cfg.PerContractMaxBudgetBytes - state.usedBytes() step := c.cfg.ExtensionBudgetBytes @@ -298,9 +212,6 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } - // New promotion candidates: contracts whose miss count crossed the - // threshold. Cap at MaxPromotedContracts; pick the highest-miss - // candidates first (greedy, simple, sufficient for v1). if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { @@ -336,8 +247,6 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui } } -// snapshotMisses atomically reads + zeros every per-contract miss -// counter. Called once per OnBlockComplete. func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { out := make(map[[32]byte]uint64) c.misses.Range(func(k, v any) bool { @@ -351,8 +260,7 @@ func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { return out } -// demoteLocked invalidates every pinned prefix for the contract. -// Caller must hold c.mu. +// demoteLocked: caller must hold c.mu. func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { for _, prefix := range state.pinnedPrefixes() { c.cache.Invalidate(prefix) @@ -366,10 +274,7 @@ func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContr } } -// promoteLocked runs the initial-view preload for a freshly-promoted -// contract. Uses the parallel wave-BFS when parallelResolve != nil, -// otherwise falls back to the serial CommitmentReader BFS. On error the -// partial pin set is rolled back. Caller must hold c.mu. +// promoteLocked: caller must hold c.mu. On error the partial pin set is rolled back. func (c *AdaptivePinController) promoteLocked( ctx context.Context, hash [32]byte, @@ -398,7 +303,6 @@ func (c *AdaptivePinController) promoteLocked( parallel: p, }, nil } - // Serial BFS fallback (no parallel resolver this block). p, err := NewContractTrunkPreload(hash[:]) if err != nil { return nil, err @@ -416,14 +320,9 @@ func (c *AdaptivePinController) promoteLocked( }, nil } -// runExtensionLocked extends a promoted contract's preload by stepBudget -// bytes. Uses the saved state's mode (parallel vs serial). Caller must -// hold c.mu. -// -// When the saved state is serial (state.preload != nil) but parallelResolve -// is available this block, we keep using serial — switching mid-contract -// would lose the queue position. Future work: convert serial-state to -// parallel-state at extension time. +// runExtensionLocked: caller must hold c.mu. Uses the saved state's mode +// (parallel vs serial); a serial state with a parallel resolver available +// keeps using serial — switching mid-contract would lose the queue position. func (c *AdaptivePinController) runExtensionLocked( ctx context.Context, state *adaptiveContractState, @@ -433,8 +332,6 @@ func (c *AdaptivePinController) runExtensionLocked( ) error { if state.parallel != nil { if parallelResolve == nil { - // Parallel state but no resolver this block — skip the - // extension; the saved state survives for the next block. return nil } var dbBranches map[string][]byte @@ -444,14 +341,10 @@ func (c *AdaptivePinController) runExtensionLocked( _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) return err } - // Serial-BFS state. _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) return err } -// PromotedContracts returns the hashes of currently-pinned contracts. -// Snapshot — the slice may be stale by the time the caller uses it. -// Used by metrics + debug diagnostics. func (c *AdaptivePinController) PromotedContracts() [][32]byte { c.mu.Lock() defer c.mu.Unlock() @@ -462,10 +355,6 @@ func (c *AdaptivePinController) PromotedContracts() [][32]byte { return out } -// pickPromotionCandidates selects up to maxN contract hashes from -// misses whose count exceeds threshold, preferring highest-miss first. -// Linear scan — adequate for the small N we expect (typically ≤16 -// candidates per block; sort is cheap). func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { if maxN <= 0 { return nil @@ -481,7 +370,6 @@ func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN } } if len(pool) > maxN { - // Partial sort: O(N×maxN) for small maxN; avoids importing sort. for i := 0; i < maxN; i++ { best := i for j := i + 1; j < len(pool); j++ { @@ -506,6 +394,4 @@ func (c *AdaptivePinController) warnf(msg string, kv ...any) { } } -// ctx unused for now; reserved for future cancellation of in-flight -// preloads (R3 bulk preload + cancellable extension). -var _ = context.Background +var _ = context.Background // reserved for cancellation of in-flight preloads diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 310a8dd4199..2a33510bff8 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -271,16 +271,9 @@ type branchCacheEntry struct { // returned by aggTx.MeteredGetLatest / tx.GetLatest. step uint64 - // txN is the high-water txNum the cached bytes correspond to. Used by - // UnwindTo: an unwind to txNum T evicts every entry whose txN > T. - // Tagging conventions per write source: - // - sd.Flush → sd.txNum (exact, in-memory write) - // - snapshot file → file.endTxNum (step-boundary, exact under - // the no-unwind-into-snapshots invariant) - // - DB latest read → lastTxNumOfStep(step) (conservative - // high-water within the step) - // 0 means "not tracked" — entry survives any watermark (back-compat - // with callers that haven't been migrated to pass a real txN yet). + // txN is the high-water txN this entry is valid through. UnwindTo + // evicts every entry whose txN > watermark. 0 means "not tracked" + // (entry survives any watermark). txN uint64 } @@ -383,11 +376,9 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // (subject to Put updates from SD.Flush refreshing the bytes). Use for // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate -// the input after the call. -// -// txN tags the entry with its high-water validity; UnwindTo evicts a -// pinned entry when its txN exceeds the watermark. Pinning protects -// against capacity-pressure eviction, not correctness eviction. +// the input after the call. UnwindTo still evicts pinned entries when +// txN > watermark — pinning protects against capacity pressure, not +// correctness. func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return @@ -503,11 +494,8 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // Put stores branch data in the cache, replacing any existing entry // (clearing its dirty flag in the process — the new entry is fresh). // Always copies the input data so the cache owns it independently of -// caller buffer lifetime. step is the on-disk file step the bytes came -// from (0 if not tracked); txN is the high-water txNum the entry is -// valid through (0 if not tracked — entry survives any UnwindTo -// watermark); origin is a short label of the write site captured for -// divergence-detection diagnostics. +// caller buffer lifetime. origin is captured for divergence-detection +// diagnostics. See entry.txN for the txN tagging semantics. func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin string) { if isCommitmentStateKey(prefix) { return @@ -594,31 +582,16 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.tail.Delete(prefix) } -// UnwindTo evicts every cache entry whose txN > maxValidTxN across all -// tiers (root, pinned, LRU tail). Returns the number of entries evicted. -// -// This is the txN-tagged-cache invalidation primitive: a single watermark -// pass over the cache, no per-key changeset required. Entries with txN=0 -// ("not tracked") are NEVER evicted by the watermark — they're treated -// as permanently valid (callers that don't tag pay no watermark cost, -// preserving back-compat for migration). -// -// Pinned entries ARE evicted by watermark — pinning protects against -// capacity-pressure eviction, not correctness eviction. -// -// Concurrency: safe to call alongside concurrent reads. Writes during -// UnwindTo may produce a slightly larger effective working set than the -// post-call watermark suggests (a Put racing with the scan may insert an -// entry the scan already passed), but every such Put carries its own txN -// and will be caught by the next UnwindTo if the entry violates the -// invariant. +// UnwindTo evicts every cache entry whose txN > maxValidTxN across +// root, pinned, and LRU tail. Returns the number of entries evicted. +// Safe to call alongside concurrent reads; a Put racing with the scan +// may insert an entry the scan already passed, but the next UnwindTo +// will catch it. func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { - // Root tier — single slot, atomic check-and-clear. if entry := c.root.Load(); entry != nil && entry.txN > maxValidTxN { c.root.Store(nil) evicted++ } - // Pinned tier — iterate and delete in place. c.pinned.Range(func(hash uint64, entry *branchCacheEntry) bool { if entry != nil && entry.txN > maxValidTxN { c.pinned.DeleteByHash(hash) @@ -626,7 +599,6 @@ func (c *BranchCache) UnwindTo(maxValidTxN uint64) (evicted int) { } return true }) - // LRU tail — iterate and delete in place. c.tail.Range(func(hash uint64, entry *branchCacheEntry) bool { if entry != nil && entry.txN > maxValidTxN { c.tail.DeleteByHash(hash) diff --git a/execution/commitment/branch_decode.go b/execution/commitment/branch_decode.go index 0e4c6b5ccc9..10093d92111 100644 --- a/execution/commitment/branch_decode.go +++ b/execution/commitment/branch_decode.go @@ -22,42 +22,23 @@ import ( "math/bits" ) -// BranchMaps captures the three branch-level bitmasks decoded alongside the -// per-cell payload. Held separately from the cells slice so callers can apply -// them into trie state without struct-assigning the whole [16]cell array. +// BranchMaps captures the three branch-level bitmasks decoded alongside +// the per-cell payload. type BranchMaps struct { - Bitmap uint16 // present-children bitmap (the canonical map encoded in the branch) - TouchMap uint16 // children that were touched in this branch (set by caller per deleted flag) + Bitmap uint16 // present-children bitmap (canonical encoded map) + TouchMap uint16 // children touched in this branch (per the deleted flag) AfterMap uint16 // children present after this commitment step } -// DecodeBranchInto parses the on-disk encoded form of a branch (as returned -// by ctx.Branch / readBranchAndCheckForFlushing with the leading 2-byte -// touch-map stripped) and populates the supplied cells array. +// DecodeBranchInto parses the on-disk encoded form of a branch into cells. +// branchData must already have the leading 2-byte touch-map prefix stripped. // -// branchData must already have the leading touch-map prefix stripped — the -// raw bytes returned from Branch() include it; callers strip it before -// invoking this function (matching the unfoldBranchNode call pattern). +// deleted=true → touchMap=bitmap, afterMap=0 (touched-but-not-present-after). +// deleted=false → touchMap=0, afterMap=bitmap (present-after). // -// deleted controls the touchMap/afterMap convention: -// - deleted=true → all decoded cells are touched-but-not-present-after -// (touchMap = bitmap, afterMap = 0) -// - deleted=false → all decoded cells are present-after (touchMap = 0, -// afterMap = bitmap) -// -// Returns the three branch-level maps for the caller to apply into their -// own state machine. The cells array is mutated in place. -// -// This is a PURE decode — it does NOT call deriveHashedKeys on the cells. -// Trie callers (HexPatriciaHashed.unfoldBranchNode) follow this with their -// own loop calling deriveHashedKeys per cell, since they have the keccak -// scratch buffer. Cache callers can skip deriveHashedKeys entirely until -// the cell is consumed by the trie. -// -// This function is the canonical decoder for branch data. Both -// unfoldBranchNode (which feeds the in-memory grid for fold/unfold) and -// future cache populators consume branches via this same path so the -// encoded→cells transformation lives in one place. +// Pure decode — does NOT call deriveHashedKeys on the cells. Trie callers do +// that themselves (they have the keccak scratch buffer); cache callers can +// defer it until the cell is consumed by the trie. func DecodeBranchInto( branchData []byte, deleted bool, diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 612b06080f9..741f2dea700 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -100,11 +100,9 @@ type Trie interface { SetCapture(capture []string) GetCapture(truncate bool) []string EnableCsvMetrics(filePathPrefix string) - // SetBranchCache attaches a BranchCache instance for branch read/write - // caching across the trie + branchEncoder. ConcurrentPatriciaHashed - // implementations propagate the same instance to all mounts so the - // concurrency contract documented in branch_cache.go is respected - // (single shared cache, partitioned-by-prefix writers). + // SetBranchCache attaches the shared cache used by trie + branchEncoder. + // ConcurrentPatriciaHashed propagates the same instance to all mounts — + // see branch_cache.go for the concurrency contract. SetBranchCache(*BranchCache) // Variant returns commitment trie variant @@ -154,12 +152,9 @@ const ( VariantConcurrentHexPatricia TrieVariant = "hex-concurrent-patricia-hashed" ) -// InitializeTrieAndUpdates constructs the trie + updates buffer for a -// SharedDomainsCommitmentContext. The branchCache argument is the -// long-lived (aggregator-scope) cache shared across SDs — pass the cache -// returned by BranchCacheProvider on the AggregatorRoTx. When nil (test -// helpers, isolated benchmarks), a fresh per-init cache is created so the -// trie still has a valid cache to read/write through. +// InitializeTrieAndUpdates constructs the trie + updates buffer. branchCache +// should be the aggregator-scope shared cache; nil falls back to a fresh +// per-init cache (test helpers, isolated benchmarks). func InitializeTrieAndUpdates(tv TrieVariant, mode Mode, tmpdir string, branchCache *BranchCache) (Trie, *Updates) { if branchCache == nil { branchCache = NewBranchCache(DefaultBranchCacheTailCapacity) @@ -364,10 +359,7 @@ type BranchEncoder struct { deferUpdates bool deferred []*DeferredBranchUpdate pendingPrefixes *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates - // branchCache, when set, receives mark-dirty-before-encode + Put-after- - // canonical-write so the cache stays consistent with ctx.PutBranch. - // Set via HexPatriciaHashed.SetBranchCache (which propagates here). - branchCache *BranchCache + branchCache *BranchCache // set via HexPatriciaHashed.SetBranchCache } func NewBranchEncoder(sz uint64) *BranchEncoder { @@ -583,11 +575,9 @@ func (be *BranchEncoder) CollectUpdate( var prev []byte var err error - // Mark the BranchCache entry dirty BEFORE doing the encode work. Any - // concurrent warmer-style writer that races into PutIfClean for this - // prefix between now and our final Put below will see the dirty flag - // and skip — preventing a stale read from overwriting our fresh - // canonical write. See branch_cache.go's Concurrency Contract. + // MarkDirty BEFORE encode so any concurrent PutIfClean for this prefix + // skips — preventing a stale read from overwriting our canonical write. + // See branch_cache.go's Concurrency Contract. if be.branchCache != nil { be.branchCache.MarkDirty(prefix) } @@ -617,13 +607,9 @@ func (be *BranchEncoder) CollectUpdate( if err = ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil { return err } - // BranchCache update happens lazily: ctx.PutBranch writes only to sd.mem, - // which masks any cached value at this prefix for the rest of the block. - // On sd.Flush, sd.mem is moved to MDBX and BranchCache is invalidated + - // repopulated for the affected prefixes (see SD.Flush). The MarkDirty - // above protects against concurrent warmer-style writers between now and - // that flush. Single writer per prefix per fold invariant means no - // concurrent Put races on this key. + // No cache Put here: ctx.PutBranch writes to sd.mem which masks this + // prefix for the rest of the block; BranchCache is invalidated and + // repopulated at SD.Flush. if be.metrics != nil { be.metrics.updateBranch.Add(1) } @@ -657,13 +643,9 @@ func (be *BranchEncoder) CollectDeferredUpdate( be.ClearDeferred() } - // Mark BranchCache entry dirty BEFORE the deferred enqueue. Mirrors - // the CollectUpdate path so that both immediate and deferred encoder - // entries consistently invalidate the cache. Concurrent warmer-style - // writers calling PutIfClean for this prefix between now and the - // eventual ApplyDeferredBranchUpdates write see the dirty flag and - // skip — preventing a stale read from racing the deferred write. - // See branch_cache.go's Concurrency Contract. + // MarkDirty as in CollectUpdate — the deferred write happens later via + // ApplyDeferredBranchUpdates but the cache must skip concurrent + // PutIfClean writers in the interim. if be.branchCache != nil { be.branchCache.MarkDirty(prefix) } @@ -1823,9 +1805,6 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, var prevKey []byte err := t.etl.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error { - // (previously called warmuper.Cache().EvictPlainKey(v) here to drop - // stale account/storage entries; the warmer's cache was removed - // alongside WarmupCache so there's nothing to evict.) // Copy into arena since ETL may reuse buffers hk := t.arenaAlloc(k) pk := t.arenaAlloc(v) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 874c5f651fa..4c50d6bad3a 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -42,17 +42,10 @@ type sd interface { Trace() bool CommitmentCapture() bool - // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) - // for one key. Used by the BranchCache divergence-detection probe - // to localise which state layer holds bytes that diverge from - // cache. Read-only. + // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one + // key — BranchCache divergence-detection probe. Read-only. ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) - // Metrics exposes the per-SD DomainMetrics so callers can read - // per-domain (cache, db, file) read counters. Used by the - // cache-fp log line to break the aggregate `files=N` count down - // per domain (Storage value loads vs Commitment branch reads - // vs Account loads). Metrics() *changeset.DomainMetrics } @@ -190,11 +183,9 @@ func (sdc *SharedDomainsCommitmentContext) EnableCsvMetrics(filePathPrefix strin sdc.patriciaTrie.EnableCsvMetrics(filePathPrefix) } -// NewSharedDomainsCommitmentContext constructs a per-SharedDomains -// commitment context. branchCache is the aggregator-scope cache shared -// across all SDs derived from the same Aggregator; pass nil from test -// helpers that don't have an aggregator (a fresh per-init cache will be -// created inside InitializeTrieAndUpdates as a fallback). +// NewSharedDomainsCommitmentContext: branchCache is the aggregator-scope +// shared cache; nil from test helpers falls back to a per-init cache inside +// InitializeTrieAndUpdates. func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant commitment.TrieVariant, tmpDir string, branchCache *commitment.BranchCache) *SharedDomainsCommitmentContext { ctx := &SharedDomainsCommitmentContext{ sharedDomains: sd, @@ -326,7 +317,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context keysPerSec = uint64(float64(updateCount) / took.Seconds()) } log.Debug("[commitment] processed", "block", blockNum, "txNum", txNum, "keys", common.PrettyCounter(updateCount), "keys/s", common.PrettyCounter(keysPerSec), "mode", sdc.updates.Mode(), "spent", took, "rootHash", hex.EncodeToString(rootHash)) - }() if updateCount == 0 { rootHash, err = sdc.patriciaTrie.RootHash() @@ -410,11 +400,6 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context }() } - // Note: a previous EnableWarmupCache(false)+defer(true) guard lived here - // to ensure RecordingContext saw every read during trace capture. Now - // that the Go-side WarmupCache is gone, every read already goes through - // ctx.* (observable to the recorder by construction); no guard needed. - var warmupConfig commitment.WarmupConfig var drainCollectors func() []*etl.Collector if sdc.paraTrieDB != nil { @@ -590,9 +575,8 @@ func (e *errorTrieContext) Storage(plainKey []byte) (*commitment.Update, error) return nil, e.err } -// KeyCommitmentState is the commitment-domain key under which the latest -// root hash and tree state are stored. Single definition lives in the -// commitment package so BranchCache can exclude it by construction. +// KeyCommitmentState aliases commitment.KeyCommitmentState — single source of +// truth so BranchCache can exclude it by construction. var KeyCommitmentState = commitment.KeyCommitmentState var ErrBehindCommitment = errors.New("behind commitment") @@ -796,11 +780,7 @@ type TrieContext struct { stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch - // probeSd / probeTx feed ProbeStateLayers — diagnostics-only fields - // populated when this TrieContext is constructed from a - // SharedDomainsCommitmentContext that has both a SharedDomains - // and a tx in scope. Both nil for read-only / test contexts where - // the probe is not available. + // Diagnostics-only — both nil for read-only / test contexts. probeSd sd probeTx kv.TemporalTx } @@ -824,11 +804,9 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } -// ProbeStateLayers samples the underlying state layers for one key — -// sd.mem, sd.parent.mem (if any), and tx-direct (MDBX) — for -// divergence-detection diagnostics. Returns empty / not-ok when -// constructed without a probe-capable SharedDomains (e.g. -// NewTrieContextRo). Read-only. +// ProbeStateLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one +// key — divergence diagnostics. Returns empty / not-ok when constructed +// without a probe-capable SharedDomains (e.g. NewTrieContextRo). func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { if sdc.probeSd == nil { return @@ -836,10 +814,8 @@ func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, par return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) } -// SiteIdentity returns a stable string identifying the underlying -// SharedDomains pointer. Used by cache write sites to tag entries with -// the SD lineage that produced them, so divergence diagnostics can tell -// parent-SD writes apart from fork-SD writes when the cache is shared. +// SiteIdentity tags cache entries with the SD lineage that produced them so +// divergence diagnostics can tell parent-SD writes from fork-SD writes. func (sdc *TrieContext) SiteIdentity() string { return fmt.Sprintf("sd=%p", sdc.probeSd) } diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 7de86533cf7..464f10995ed 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2968,12 +2968,8 @@ func (hph *HexPatriciaHashed) SetLeaveDeferredForCaller(leave bool) { } // Reset allows HexPatriciaHashed instance to be reused for the new commitment calculation. -// -// The BranchCache is intentionally NOT cleared here: with aggregator-scope -// lifetime the cache must survive between commitment calculations to deliver -// cross-block hits. Unwind correctness is handled by the cache's own -// txN-tagged eviction in SharedDomains.Unwind — no explicit -// invalidation is required on the commitment path. +// The aggregator-scope BranchCache is intentionally not cleared here; +// SharedDomains.Unwind handles correctness via txN-tagged eviction. func (hph *HexPatriciaHashed) Reset() { hph.root.reset() hph.rootTouched = false diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index d578d43d914..4d573ceecf7 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -16,37 +16,22 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// CommitmentReader is the read interface ContractTrunkPreload needs: -// GetLatest on the CommitmentDomain. Returns (value, step, found, err). -// Decoupled from any specific tx/aggregator type so callers can plug -// the reader in at preload time without dragging in db/state imports. +// CommitmentReader does GetLatest on the CommitmentDomain. Decoupled from +// tx/aggregator types to keep this package free of db/state imports. type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) -// estimatedEntryOverheadBytes accounts for the per-entry RAM cost -// beyond the encoded value itself: the branchCacheEntry struct -// (~80 B), maphash slot + hash (~40 B), prefix slice (~24 B header + -// content), value slice header (~24 B). Used to predict whether -// pinning the next entry would exceed the configured budget. +// estimatedEntryOverheadBytes is the per-entry RAM cost beyond the encoded +// value itself: branchCacheEntry (~80 B), maphash slot + hash (~40 B), +// prefix slice (~24 B header + content), value slice header (~24 B). const estimatedEntryOverheadBytes = 168 -// pathDepth pairs a nibble path with the trie depth it terminates at. -// Used as the BFS queue element. type pathDepth struct { path []byte depth int } -// ContractTrunkPreload holds the resumable state of a BFS preload for -// one contract's storage subtree. Created by NewContractTrunkPreload -// and advanced incrementally via Run; this lets callers split the -// preload into an initial sync view (small budget for d=64-67) and -// per-block extensions (additional budget while the contract stays -// hot), instead of burst-loading the full budget at promotion time. -// -// Tracks its pinned prefixes so callers can Invalidate the contract's -// pin set on demotion (PinnedPrefixes returns the slice). -// -// Caller MUST NOT use the same instance from multiple goroutines. +// ContractTrunkPreload holds the resumable state of a BFS preload for one +// contract's storage subtree. Not goroutine-safe. type ContractTrunkPreload struct { contractHash []byte queue []pathDepth @@ -56,9 +41,8 @@ type ContractTrunkPreload struct { maxDepthReached int } -// NewContractTrunkPreload constructs a preload state seeded at depth 64 -// (storage subtree root) for the given contract. The contract's -// storage subtree path is keccak256(address) — 32 bytes / 64 nibbles. +// NewContractTrunkPreload seeds a preload state at depth 64 (storage +// subtree root: keccak256(address), 32 bytes / 64 nibbles). func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) { if len(contractHash) != 32 { return nil, fmt.Errorf("NewContractTrunkPreload: contractHash must be 32 bytes, got %d", len(contractHash)) @@ -75,19 +59,10 @@ func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) }, nil } -// Run advances the BFS preload, pinning branches into cache until -// either the additional budget is exhausted or the queue is empty. -// Returns the number of entries pinned during THIS call (not the -// cumulative total — see PinnedTotal), whether the queue is now -// empty (preload fully complete), and any error from the reader. -// -// Calling Run repeatedly with small budgets implements the phased -// load: an initial Run with the InitialBudget covers depths 64-67; -// each subsequent Run extends BFS one chunk further. Per-block -// callers should size the chunk to fit within inter-block idle time. -// -// On reader error the partial pin set survives; the queue position -// is preserved so a retry on the next Run picks up where it left off. +// Run advances the BFS one chunk, pinning branches until additionalBudgetBytes +// is exhausted or the queue is empty. Returns entries pinned THIS call, whether +// the preload is now complete, and any reader error. On error the partial pin +// set and queue position are preserved for a retry on the next Run. func (p *ContractTrunkPreload) Run( additionalBudgetBytes int, reader CommitmentReader, @@ -119,15 +94,12 @@ func (p *ContractTrunkPreload) Run( entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) if chunkUsedBytes+entryCost > additionalBudgetBytes { - // Re-queue the head so a future Run picks it up. p.queue = append([]pathDepth{head}, p.queue...) break } cache.PinEntry(prefix, v, step, 0, "preload-trunk") - // Copy prefix because nibbles.HexToCompact may return a slice - // over a reused buffer; we need a stable handle for later - // Invalidate on demotion. + // HexToCompact may alias a reused buffer; copy for a stable Invalidate handle. prefixCopy := make([]byte, len(prefix)) copy(prefixCopy, prefix) p.pinnedPrefixes = append(p.pinnedPrefixes, prefixCopy) @@ -142,9 +114,7 @@ func (p *ContractTrunkPreload) Run( "used_mb", (p.usedBytes+chunkUsedBytes)/(1<<20)) } - // Decode bitmap and queue children. Branch encoding: - // 2-byte touchMap || 2-byte bitmap || per-child data. Only - // queue children that actually exist in the trie (bit set). + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. if len(v) < 4 { continue } @@ -165,35 +135,16 @@ func (p *ContractTrunkPreload) Run( return chunkPinned, len(p.queue) == 0, nil } -// PinnedTotal returns the cumulative number of branches pinned by -// this preload state across all Run calls. -func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } - -// UsedBytes returns the cumulative byte cost of this preload's pin set. -func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } - -// QueueRemaining returns the number of paths still queued for descent. -// Zero means the preload is complete; further Run calls are no-ops. -func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } - -// MaxDepthReached returns the deepest trie depth pinned so far. +func (p *ContractTrunkPreload) PinnedTotal() int { return p.pinned } +func (p *ContractTrunkPreload) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreload) QueueRemaining() int { return len(p.queue) } func (p *ContractTrunkPreload) MaxDepthReached() int { return p.maxDepthReached } -// PinnedPrefixes returns the prefix slices this preload has added to -// the cache so far. The adaptive controller uses this to Invalidate -// the contract's pin set on demotion. The returned slice aliases -// internal storage; do not mutate. +// PinnedPrefixes returns slices aliasing internal storage — do not mutate. func (p *ContractTrunkPreload) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } +func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } -// ContractHash returns the contract this preload targets. -func (p *ContractTrunkPreload) ContractHash() []byte { return p.contractHash } - -// PreloadContractTrunk runs the full BFS preload for a contract in a -// single call, bounded by ramBudgetBytes. Convenience wrapper around -// NewContractTrunkPreload + Run for callers that don't need phased -// loading. Existing one-shot trigger paths (PIN_CONTRACT_TRUNKS env -// hook) use this; the adaptive layer holds a *ContractTrunkPreload -// and calls Run incrementally. +// PreloadContractTrunk is the one-shot wrapper around NewContractTrunkPreload + Run. func PreloadContractTrunk( contractHash []byte, ramBudgetBytes int, diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 1e8ff8a464f..abf7f937278 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -19,38 +19,28 @@ import ( ) // BatchBranchResolver resolves a batch of compact-encoded CommitmentDomain -// keys to their (already-dereferenced) branch-node values, reading from the -// file layer only — no MDBX cursor, hence no per-key cgo boundary crossing. -// The implementation is expected to fan the batch out across worker RoTxs; -// keys are passed in ascending key order so a contiguous-slice partition maps -// to contiguous file regions (drives page-cache readahead). Results MUST be -// returned in the same order as keys, with vals[i] == nil for keys not present -// in the file layer. -// -// Decoupled (callback form) so the commitment package needn't import db/state. +// keys to branch-node values, reading from the file layer only (no MDBX cursor, +// no per-key cgo crossing). Keys are passed in ascending key order so a +// contiguous-slice partition maps to contiguous file regions (page-cache +// readahead). Results MUST be returned in the same order as keys, with +// vals[i] == nil for keys not present. type BatchBranchResolver func(keys [][]byte) (vals [][]byte, err error) -// estimatedEntryCost is the predicted RAM cost of pinning a (key, value) branch -// entry — see estimatedEntryOverheadBytes in preload.go. func estimatedEntryCost(key, value []byte) int { return estimatedEntryOverheadBytes + len(key) + len(value) } -// minEntryBytes is a true lower bound on estimatedEntryCost for a storage-trunk +// minEntryBytes: true lower bound on estimatedEntryCost for a storage-trunk // branch (33 = shortest HexToCompact key at depth >= 64; value may be empty). -// Used to bound a wave's *file fetch*: capping at remaining/minEntryBytes+1 -// never under-fetches, so the budget is guaranteed exhausted inside the wave. +// Bounds a wave's file fetch so the budget is guaranteed exhausted inside it. const minEntryBytes = estimatedEntryOverheadBytes + 33 -// maxStorageTrunkDepth is the deepest total nibble path a storage branch can -// have: 64 (account path) + 64 (keccak256(slot)) = 128. +// maxStorageTrunkDepth: 64 (account path) + 64 (keccak256(slot)) = 128. const maxStorageTrunkDepth = 128 -// pathKey is a single frontier entry: the nibble path (1 byte / nibble) and the -// HexToCompact-encoded CommitmentDomain key. type pathKey struct { path []byte // nibble path (1 byte / nibble) - key []byte // HexToCompact(path) — the CommitmentDomain key + key []byte // HexToCompact(path) } func toPathKey(path []byte) pathKey { @@ -60,37 +50,17 @@ func toPathKey(path []byte) pathKey { return pathKey{path: path, key: kc} } -// ContractTrunkPreloadParallel is the resumable analogue of ContractTrunkPreload -// (the serial BFS via CommitmentReader). It walks the contract's storage subtree -// breadth-first one depth-level ("wave") at a time and resolves missing branches -// through a batched, file-only BatchBranchResolver — no MDBX boundary crossings -// in the hot path. Each Run advances zero or more complete waves bounded by a -// per-call step budget; partial waves can be truncated by the file-fetch cap to -// fit the budget, but the truncated tail's siblings are skipped (BFS at the -// frontier is then resumed at depth+1 from the children of whatever was pinned). -// -// Mirrors ContractTrunkPreload's API surface (Run/PinnedTotal/UsedBytes/ -// QueueRemaining/MaxDepthReached/PinnedPrefixes/ContractHash) so it slots in to -// adaptive_pin.go as a drop-in replacement without changing the controller's -// promote/extend/demote loop. +// ContractTrunkPreloadParallel is the wave-BFS analogue of ContractTrunkPreload. +// It walks one depth-level per wave and resolves missing branches through a +// batched, file-only BatchBranchResolver (no MDBX in the hot path). Each Run +// advances zero or more waves bounded by stepBudgetBytes; partial waves are +// truncated to fit the budget and resumed on the next Run. // -// Correctness: dbBranches values shadow file values for the same key (DB is -// authoritative for steps not yet flushed to files), so callers extending on a -// live node should pass the freshest MDBX-resident set per Run. With an empty -// dbBranches Run is a pure file-only preload — correct only from a cold snapshot -// or when MDBX/SD's flush-callback has already populated the cache with any -// freshly-written branches that would otherwise shadow the file values. +// dbBranches shadows file values for the same key — DB is authoritative for +// steps not yet flushed to files. Pass nil for cold-snapshot / file-only mode. // -// Caller MUST NOT use the same instance from multiple goroutines. The -// BatchBranchResolver is passed PER Run rather than held on the struct so -// callers can supply a fresh tx-scoped resolver each call (e.g. adaptive -// controller building a resolver from the in-flight Flush tx every block). -// -// Resumability: when a step budget cuts a wave mid-iteration, the un-processed -// items at the current depth are preserved in `frontier`; the children of -// the items that did get pinned this wave land in `pendingChildren` at depth -// nextDepth+1. The next Run finishes the current depth first, then advances. -// Once a wave fully completes, frontier := pendingChildren and nextDepth++. +// Not goroutine-safe. The resolver is passed per-Run (not held) so callers +// can supply a fresh tx-scoped resolver each block. type ContractTrunkPreloadParallel struct { contractHash []byte frontier []pathKey // paths to process at depth = nextDepth @@ -103,10 +73,7 @@ type ContractTrunkPreloadParallel struct { dbHitsPinned int } -// NewContractTrunkPreloadParallel seeds a resumable preload state at depth 64 -// (storage subtree root) for the given contract. The resolver is passed -// per-Run (not stored on the state) so callers can hand a fresh tx-scoped -// resolver to each call. +// NewContractTrunkPreloadParallel seeds a preload at depth 64 (storage subtree root). func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreloadParallel, error) { if len(contractHash) != 32 { return nil, fmt.Errorf("NewContractTrunkPreloadParallel: contractHash must be 32 bytes, got %d", len(contractHash)) @@ -121,20 +88,9 @@ func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreload }, nil } -// Run advances the wave-BFS, pinning branches into cache until either the step -// budget is exhausted, the frontier is empty (preload complete), or -// maxStorageTrunkDepth is reached. Returns newlyPinned during THIS call (not -// cumulative — see PinnedTotal), whether the preload is now complete, and any -// resolver error. -// -// dbBranches is consulted per wave (MDBX-resident overlays — values shadow file -// values, freshest). Pass nil for file-only mode (correct from a cold snapshot -// or when SD.Flush's cache callback has already populated freshly-written -// branches into the cache that would otherwise need to come from dbBranches). -// -// On resolver error the partial pin set survives; the wave position is -// preserved so a retry on the next Run picks up where it left off (at the same -// p.nextDepth). +// Run advances the wave-BFS until stepBudgetBytes is exhausted, the frontier +// is empty, or maxStorageTrunkDepth is reached. On resolver error the partial +// pin set and wave position survive for retry on the next Run. func (p *ContractTrunkPreloadParallel) Run( stepBudgetBytes int, dbBranches map[string][]byte, @@ -156,8 +112,7 @@ func (p *ContractTrunkPreloadParallel) Run( chunkPinned := 0 budgetHit := false - // pin records the entry, advances accounting, queues the branch's - // children. Returns false if the entry didn't fit (step budget exhausted). + // pin records the entry and queues its children. Returns false on budget hit. pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { cost := estimatedEntryCost(pk.key, v) if p.usedBytes+cost > stepCap { @@ -195,14 +150,9 @@ func (p *ContractTrunkPreloadParallel) Run( for !budgetHit && p.nextDepth <= maxStorageTrunkDepth && len(p.frontier) > 0 { depth := p.nextDepth - // Ascending key order so a contiguous-slice partition of the file batch - // is contiguous-in-file (page-cache readahead). BFS append order is - // already key-sorted (HexToCompact packing is monotone for same-length - // paths), but sort defensively — a few k entries / wave, cheap. + // Ascending key order so the file-batch partition is contiguous-in-file. sort.Slice(p.frontier, func(i, j int) bool { return bytes.Compare(p.frontier[i].key, p.frontier[j].key) < 0 }) - // Split this wave into DB-resident hits (have the value) and file-misses - // (need a fetch). Both stay sorted (sub-sequences of the sorted frontier). var dbHits []pathKey var dbVals [][]byte var fileMiss []pathKey @@ -217,10 +167,7 @@ func (p *ContractTrunkPreloadParallel) Run( } } - // Cap the *file fetch* at what the step budget can absorb after the - // DB-hits (which we pin first). minEntryBytes is a true lower bound, - // so this never under-fetches; if the cap truncates the wave, the - // un-fetched tail is preserved in p.frontier for the next Run. + // Cap the file fetch by what the budget can absorb after dbHits. var fileMissDeferred []pathKey if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { fileMissDeferred = fileMiss @@ -238,7 +185,6 @@ func (p *ContractTrunkPreloadParallel) Run( } fileVals, err = resolve(keys) if err != nil { - // State unchanged — retry on next Run picks up at same depth. return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", depth, err) } if len(fileVals) != len(keys) { @@ -246,10 +192,6 @@ func (p *ContractTrunkPreloadParallel) Run( } } - // Pin in dbHits-then-fileMiss order. Children of pinned items accumulate - // in p.pendingChildren (depth nextDepth+1). On budget hit mid-iteration, - // the un-processed remainder of dbHits/fileMiss + the deferred - // fileMissDeferred all become the new frontier (still at nextDepth). dbHitStop := len(dbHits) for i, pk := range dbHits { if !pin(pk, dbVals[i], depth, &p.pendingChildren) { @@ -263,7 +205,7 @@ func (p *ContractTrunkPreloadParallel) Run( for i, pk := range fileMiss { v := fileVals[i] if v == nil { - continue // not in the file layer either + continue } if !pin(pk, v, depth, &p.pendingChildren) { fileMissStop = i @@ -273,26 +215,18 @@ func (p *ContractTrunkPreloadParallel) Run( } if budgetHit { - // Preserve un-pinned items at the current depth for the next Run. - // Resolved-but-not-pinned file values are discarded (callable resolver - // will be re-invoked on the next Run for these keys). + // Preserve un-pinned items at current depth; pendingChildren stays + // at depth+1 for when this depth is drained on a future Run. rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) rest = append(rest, dbHits[dbHitStop:]...) rest = append(rest, fileMiss[fileMissStop:]...) rest = append(rest, fileMissDeferred...) p.frontier = rest - // Don't advance nextDepth — pendingChildren stays at depth+1 for - // when this depth is fully drained on a future Run. break } - // Wave fully processed. Anything in fileMissDeferred (won't have any - // since !budgetHit ⇒ no truncation) goes back to frontier; pending - // children promote to the next wave. if len(fileMissDeferred) > 0 { - // Shouldn't happen — if we got here without budgetHit, no truncation - // occurred. Defensive: re-queue them at the current depth so we - // don't lose data. + // Defensive: !budgetHit should mean no truncation. Re-queue at current depth. p.frontier = fileMissDeferred } else { p.frontier = p.pendingChildren @@ -318,43 +252,21 @@ func (p *ContractTrunkPreloadParallel) Run( return chunkPinned, queueEmpty, nil } -// PinnedTotal returns the cumulative number of branches pinned by this preload -// state across all Run calls. -func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } - -// UsedBytes returns the cumulative byte cost of this preload's pin set. -func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } +func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } +func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } +func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } -// QueueRemaining returns the number of paths still queued for descent -// (un-processed at the current depth + accumulated children at the next). -// Zero means the preload is complete; further Run calls are no-ops. func (p *ContractTrunkPreloadParallel) QueueRemaining() int { return len(p.frontier) + len(p.pendingChildren) } -// MaxDepthReached returns the deepest trie depth pinned so far. -func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } - -// PinnedPrefixes returns the prefix slices this preload has added to the cache -// so far. The adaptive controller uses this to Invalidate the contract's pin -// set on demotion. The returned slice aliases internal storage; do not mutate. +// PinnedPrefixes returns slices aliasing internal storage — do not mutate. func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } -// ContractHash returns the contract this preload targets. -func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } - -// DbHitsPinned returns the cumulative number of branches that were pinned from -// the dbBranches overlay (rather than via the file resolver). Diagnostic only. -func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } - -// PreloadContractTrunkParallel pins contract H's storage-trunk branches into -// cache in a single call, bounded by ramBudgetBytes. Convenience wrapper around -// NewContractTrunkPreloadParallel + Run for callers that don't need phased -// loading. The startup PIN_CONTRACT_TRUNKS hook uses this; the adaptive layer -// holds a *ContractTrunkPreloadParallel and calls Run incrementally. -// -// dbBranches is the MDBX-resident set — values shadow file values for the same -// key. Pass nil/empty for file-only mode (correct only from a cold snapshot). +// PreloadContractTrunkParallel is the one-shot wrapper around +// NewContractTrunkPreloadParallel + Run. func PreloadContractTrunkParallel( contractHash []byte, ramBudgetBytes int, diff --git a/execution/commitment/preload_ranges.go b/execution/commitment/preload_ranges.go index 7883ad95681..3e2f9b8406d 100644 --- a/execution/commitment/preload_ranges.go +++ b/execution/commitment/preload_ranges.go @@ -10,10 +10,8 @@ package commitment import "github.com/erigontech/erigon/execution/commitment/nibbles" -// NextSubtree returns the smallest key K' that is greater than every key having -// `in` as a prefix — i.e. the exclusive upper bound for a prefix-range scan over -// `in`. Returns nil if `in` is all 0xff (no such K'). Mirrors db/kv.NextSubtree; -// inlined to keep this package's import set minimal. +// NextSubtree: exclusive upper bound of a prefix-range scan over `in`. Returns +// nil if `in` is all 0xff. Mirrors db/kv.NextSubtree (inlined to keep imports minimal). func NextSubtree(in []byte) []byte { r := make([]byte, len(in)) copy(r, in) @@ -50,8 +48,7 @@ func ContractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, return evenFrom, evenTo, oddFrom, oddTo } -// ContractNibbles expands a 32-byte contract hash to its 64-nibble path -// (one nibble per byte, high nibble first). +// ContractNibbles expands a 32-byte hash to its 64-nibble path (high nibble first). func ContractNibbles(contractHash []byte) []byte { out := make([]byte, len(contractHash)*2) for i, b := range contractHash { diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go index 82c6b2e26fc..b16996278ee 100644 --- a/execution/commitment/trunk_pin_metrics.go +++ b/execution/commitment/trunk_pin_metrics.go @@ -12,38 +12,25 @@ import ( "github.com/erigontech/erigon/diagnostics/metrics" ) -// Prometheus metrics for the storage-trunk pin layer. Per-contract -// labels intentionally omitted — cardinality risk is real (mainnet -// could see hundreds of promoted contracts over a process lifetime) -// and the structured [adaptive-pin] log line gives per-contract -// detail when needed for debugging. +// Per-contract labels omitted to keep cardinality bounded; per-contract +// detail is in the [adaptive-pin] structured log line. var ( - // BranchCache pinned-tier counters. Surface the existing - // PinnedStats hits/misses/entries so dashboards can reason about - // pin effectiveness without scraping logs. mxPinnedHits = metrics.GetOrCreateCounter("commitment_branchcache_pinned_hits_total") mxPinnedMisses = metrics.GetOrCreateCounter("commitment_branchcache_pinned_misses_total") mxPinnedEntries = metrics.GetOrCreateGauge("commitment_branchcache_pinned_entries") - // AdaptivePinController lifecycle counters. mxAdaptivePromoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_promoted_total") mxAdaptiveExtended = metrics.GetOrCreateCounter("commitment_adaptive_pin_extended_total") mxAdaptiveDemoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_demoted_total") mxAdaptiveActive = metrics.GetOrCreateGauge("commitment_adaptive_pin_active_contracts") - // Preload-side counters for diagnostics. Per-contract preload - // duration is too high-cardinality for a histogram label; total - // duration spent in preload is the aggregate proxy. mxPreloadDurationSecondsTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_duration_seconds_total") mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") ) -// PublishMetrics snapshots the cache's current counters into the -// Prometheus registry. Counters are monotonic-add so the cache tracks -// last-published values internally and emits deltas. Gauges Set -// absolute. Call periodically from the host (e.g. once per SD.Flush); -// the once-per-batch cadence avoids hot-path cost. +// PublishMetrics emits counter deltas (last-published tracked internally) and +// sets gauges absolute. Call once per SD.Flush — once-per-batch avoids hot-path cost. func (c *BranchCache) PublishMetrics() { hits := c.pinnedHits.Load() misses := c.pinnedMisses.Load() diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index f71a73fa312..2e334ca6833 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -30,21 +30,16 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// Warmer outcome counters — measurement scaffolding to size the -// hit-rate of warmer branch reads. Hit means branchFromCacheOrDB -// returned >= 4 bytes (a real branch). Empty means it returned -// nothing or too short to parse (no branch at this depth in the -// on-disk trie). Used to bound the value of bypassing the xorfilter -// for the warmer specifically — high hit ratio implies the filter -// is mostly overhead in this call path. +// Warmer branch-read outcome counters. Hit: branchFromCacheOrDB returned +// >= 4 bytes; Empty: returned nothing or unparseable. Used to size the +// value of bypassing the xorfilter in this call path. var ( warmerBranchHitCount atomic.Uint64 warmerBranchEmptyCount atomic.Uint64 ) -// WarmerBranchOutcomeStats returns process-cumulative counts of -// branch-read outcomes from the warmer's depth walk. To get a -// per-block delta, snapshot before/after. +// WarmerBranchOutcomeStats returns process-cumulative counts. Snapshot +// before/after for per-block deltas. func WarmerBranchOutcomeStats() (hit, empty uint64) { return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() } @@ -111,11 +106,6 @@ func NewWarmuper(ctx context.Context, cfg WarmupConfig) *Warmuper { } } -// branchFromCacheOrDB reads branch data via ctx.Branch. The Go-side -// warmup cache that previously sat above this has been removed; the -// aggregator-scope BranchCache (reached through SD.GetLatest) is the -// only branch cache. The function is kept as a thin wrapper to -// preserve the warmer-side error handling shape. func (w *Warmuper) branchFromCacheOrDB(trieCtx PatriciaContext, prefix []byte) ([]byte, error) { branchData, _, err := trieCtx.Branch(prefix) return branchData, err @@ -181,16 +171,6 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep } nextNibble := int(hashedKey[depth]) - // Account/storage prefetch removed: the warmer's scope is branch - // warm-up only. If a leaf hash needs underlying account/storage - // data during fold, that data is either (a) memoized as the - // cell's stateHash, (b) carried in the Updates buffer from the - // executor, or (c) faulted on the trie's own slow path — - // signalling a memoization gap that wants fixing at the trie - // layer, not papered over by a separate prefetcher. Counters - // disk_sto / disk_acc on the [commitment][cache-fp] log line - // expose any such fall-through. - branchData = branchData[2:] // skip touch map bitmap := binary.BigEndian.Uint16(branchData[0:2]) @@ -220,11 +200,8 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep fieldBits := branchData[pos] pos++ - // If the child cell is a leaf (holds a plain key), there is no - // branch below it — stop walking. Without this check the warmer - // pays one extra recsplit + xorfilter lookup per leaf-terminating - // path, all of which return empty. Mirrors what unfold does - // implicitly by traversing the actual trie structure. + // Leaf cell — no branch below. Without this the warmer pays one + // extra recsplit + xorfilter lookup per leaf-terminating path. if cellFields(fieldBits)&(fieldAccountAddr|fieldStorageAddr) != 0 { break } diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 0623b5bcaa6..91da37f13e1 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -149,15 +149,6 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { return fmt.Errorf("failed to flush shared domains: %w", err) } - // SetHead unwinds commitment history but the BranchCache (aggregator- - // scope, shared across SDs) still holds entries from the pre-unwind - // state — notably KeyCommitmentState pointing at the original head - // blockNum. The next FCU's NewSharedDomains.SeekCommitment would hit - // the cache, see the original-head blockNum, and trip - // ErrBehindCommitment against the now-truncated TxNums index. Clear - // the cache so subsequent reads see the post-unwind MDBX state. - sd.ClearBranchCache() - if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } From 4618add1c09a8c76d2726774a0b967bf98cd521a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 27 May 2026 07:09:04 +0000 Subject: [PATCH 099/120] execution/engineapi, db/state/execctx, execution/commitment: remove orphaned cache-clear API surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SharedDomains.Unwind already evicts BranchCache entries by txN watermark via the UnwindTo path. After dropping ExecModule.SetHead's redundant clear in the previous commit, none of these stayed live; they're all removed now so callers can't reach in to wipe the cache. - EngineXTestRunner.Run: drop tester.ResetBranchCache(). The tester is reused across (fork, preAllocHash) groups; the next test's NewPayload+FCU triggers a reorg-back-to-genesis through SD.Unwind, which evicts every entry with txN above the unwound watermark. - EngineApiTester.ResetBranchCache: orphan after the above. Removed. - SharedDomains.ClearBranchCache: orphan after the SetHead removal. Removed — SD owns cache lifecycle; callers must not bypass. - HexPatriciaHashed.ClearBranchCache: orphan. Removed. BranchCache.Clear (the underlying primitive) stays — still used by the older rawdbreset.ResetExec path (#21138) for table-wipe scenarios. --- db/state/execctx/domain_shared.go | 12 -------- execution/commitment/hex_patricia_hashed.go | 11 ------- .../engineapitester/engine_api_tester.go | 29 ------------------- .../engineapitester/engine_x_test_runner.go | 5 ---- 4 files changed, 57 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index bc7e356106b..180d2d9044e 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -851,18 +851,6 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return sd.mem.Flush(ctx, tx) } -// ClearBranchCache empties the aggregator-scope commitment BranchCache. -// Use after operations that mutate commitment state outside the normal -// sd.Flush callback path — notably SetHead unwind, which truncates -// commitment domain history but does not re-write all the keys whose -// cached values are now stale. Without this call, the next FCU sees -// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. -func (sd *SharedDomains) ClearBranchCache() { - if sd.branchCache != nil { - sd.branchCache.Clear() - } -} - // TemporalDomain satisfaction func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { if tx == nil { diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 464f10995ed..2ad57cc885f 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -2977,17 +2977,6 @@ func (hph *HexPatriciaHashed) Reset() { hph.rootPresent = true } -// ClearBranchCache drops every entry in the attached BranchCache. Use on -// unwind or fork-validation paths where the cached branches diverge from -// the canonical store. No-op when no cache is attached or when this is a -// mounted subtrie (the root trie owns the clear, mounts share the same -// cache pointer). -func (hph *HexPatriciaHashed) ClearBranchCache() { - if hph.branchCache != nil && !hph.mounted { - hph.branchCache.Clear() - } -} - func (hph *HexPatriciaHashed) ResetContext(ctx PatriciaContext) { hph.ctx = ctx } diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index 1d9cd14a47d..407d1583b1f 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -41,7 +41,6 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" - dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/chain/networkname" @@ -403,34 +402,6 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs }, nil } -// ResetBranchCache drops the aggregator's in-memory branchCache. The cache is -// aggregator-scoped and survives a tx rollback, so a tester reused across -// independent EEST cases would otherwise serve commitment-trie entries -// populated by the previous test's pre-state — producing a `wrong trie root -// of block 1` on the next test's genesis→block-1 boundary. Same mechanism the -// rawdbreset.ResetExec path uses after wiping the commitment table (#21138); -// here the trigger is "moving to a new test" instead of "wiping the table". -// Safe to call at any time the tester is between tests; no-op if no -// aggregator-backed db is attached. -func (eat EngineApiTester) ResetBranchCache() { - if eat.ChainDB == nil { - return - } - hasAgg, ok := eat.ChainDB.(dbstate.HasAgg) - if !ok { - return - } - agg, ok := hasAgg.Agg().(*dbstate.Aggregator) - if !ok { - return - } - aggTx := agg.BeginFilesRo() - if bc := aggTx.BranchCache(); bc != nil { - bc.Clear() - } - aggTx.Close() -} - type EngineApiTesterInitArgs struct { Logger log.Logger DataDir string diff --git a/execution/engineapi/engineapitester/engine_x_test_runner.go b/execution/engineapi/engineapitester/engine_x_test_runner.go index 0a01116a946..aaebe201e2d 100644 --- a/execution/engineapi/engineapitester/engine_x_test_runner.go +++ b/execution/engineapi/engineapitester/engine_x_test_runner.go @@ -222,11 +222,6 @@ func (extr *EngineXTestRunner) Run(ctx context.Context, test EngineXTestDefiniti if err != nil { return err } - // Reuse the cached tester for the (fork, preAllocHash) group but reset the - // aggregator-scope branchCache so commitment-trie entries from the prior - // test don't leak into this test's genesis→block-1 boundary. Cheaper than - // evicting the whole tester (which would rebuild the genesis + node). - tester.ResetBranchCache() return extr.execute(ctx, tester, test) } From 0dc7b1bec5c23720849a15b30d2f2b39e47680ff Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 27 May 2026 08:55:11 +0000 Subject: [PATCH 100/120] execution/commitment, db/state/execctx, cmd/evm: fix 4 Copilot review issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - branch_cache: ContractHashFromPrefix mis-attributed odd-length nibble paths. HexToCompact stores the first nibble in the flag byte's low half when the path has odd parity, so prefix[1:33] read the contract hash off by one nibble. Decoder now handles both parities; the adaptive pin controller's per-contract miss tracking is now correct for storage-trunk depths >= 65 with odd parity. - branch_cache: PutIfClean and MarkDirty went through lookup(), which bumps miss counters and fires the onMiss callback. Both are write-path operations — counting them as triple-misses inflated the promotion signal for the adaptive pin controller. Added a peek() helper that walks the same tiers without side effects; PutIfClean and MarkDirty use it instead. - domain_shared: EnableParaTrieDB's once-per-cache TryClaimPreload guard also gated the per-SD adaptive controller bind. After the first SD, subsequent SDs' controllers never installed their onMiss callback, leaving the cache pointing at a closed predecessor's handler. Split: TryClaimPreload still gates the env-driven PIN_CONTRACT_TRUNKS preload (intentional fire-once), but Bind() runs on every SD. - staterunner bench mode: RunNoVerify reused the SharedDomains the preceding test.Run had already mutated, so the timed work started from a dirty post-state. Bench now opens its own tx + SD so each iteration starts from the same pre-state as the test under measure. Tests: TestContractHashFromPrefix_EvenAndOddNibblePaths covers depth 64/65/66 round-trip. TestBranchCache_PutIfClean_DoesNotCountAsMiss pins the write-path-no-fire invariant for both PutIfClean and MarkDirty. --- cmd/evm/staterunner.go | 20 ++++++- db/state/execctx/domain_shared.go | 35 +++++------- execution/commitment/branch_cache.go | 42 +++++++++++---- execution/commitment/branch_cache_test.go | 65 +++++++++++++++++++++++ 4 files changed, 128 insertions(+), 34 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index b4f34b2bec4..67c1ab494b2 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -250,8 +250,26 @@ func runStateTest(ctx *cli.Context, cfg vm.Config, fname string) ([]testResult, } } if bench { + // Run the benchmark on a fresh tx + SD so the timed work + // starts from the same pre-state every iteration; the + // preceding test.Run mutated sd in place (pre-state + + // tx execution + commitment), which would otherwise + // skew the gas/throughput numbers. + benchTx, err := db.BeginTemporalRw(context.Background()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer benchTx.Rollback() + benchSD, err := execctx.NewSharedDomains(context.Background(), benchTx, log.New()) + if err != nil { + result.Pass, result.Error = false, err.Error() + return + } + defer benchSD.Close() + _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.RunNoVerify(nil, sd, tx, st, cfg, dirs) + _, _, gasUsed, _ := test.RunNoVerify(nil, benchSD, benchTx, st, cfg, dirs) return nil, gasUsed, nil }) result.Stats = &stats diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 180d2d9044e..8136724198d 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1273,36 +1273,25 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) - // Stage-init is the right firing point for both the env-driven - // forced preload AND the adaptive controller's bind: - // (a) we have a kv.TemporalRoDB to spawn an own-tx preload goroutine - // with — avoids the cgo-pointer panic that the earlier shared-tx - // attempt produced. - // (b) we run from the stage-init path, not an engine request handler, - // so we don't block engine HTTP responses during the 3-4s preload - // window — the synchronous-from-NewSharedDomains shape caused the - // bench's first NewPayload to be dropped, breaking block validation. - // TryClaimPreload guards fire-once-per-BranchCache-lifetime. - if sd.branchCache == nil || !sd.branchCache.TryClaimPreload() { + if sd.branchCache == nil { return } - // Operator-override path: PIN_CONTRACT_TRUNKS forces specific - // contracts to be pinned regardless of adaptive policy. Runs - // first so its pin set is in place before the controller binds. - if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { - triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + // PIN_CONTRACT_TRUNKS preload is fire-once-per-BranchCache-lifetime: + // the cache is aggregator-scope and the preload's IO cost is paid up + // front; subsequent SDs reuse the already-populated pins. + if sd.branchCache.TryClaimPreload() { + if pinList := dbg.EnvString("PIN_CONTRACT_TRUNKS", ""); pinList != "" { + triggerTrunkPreload(context.Background(), sd.branchCache, db, pinList, sd.logger) + } } - // Bind the adaptive controller to the cache so it starts - // observing miss pressure for un-forced contracts. The reader - // for promote/extend preloads is constructed per-call from the - // in-flight Flush tx (see SD.Flush). + // Bind the adaptive controller on every SD: the controller is + // per-SD and Bind installs the cache's onMiss callback. Without + // this, SDs created after the first have inert controllers and + // the cache's onMiss may point at a closed predecessor. if sd.adaptivePinController != nil { sd.adaptivePinController.Bind() - sd.logger.Info("[adaptive-pin] enabled", - "max_promoted", commitment.DefaultAdaptivePinControllerConfig().MaxPromotedContracts, - "per_contract_max_mb", commitment.DefaultAdaptivePinControllerConfig().PerContractMaxBudgetBytes/(1<<20)) } } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 2a33510bff8..9767ed54dfc 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -330,8 +330,6 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { c.rootHits.Add(1) return entry, true } - // Check pinned tier before tail. Pinned entries never evict and - // are fed by PinEntry (preload) + Put updates that preserve pin. if entry, ok := c.pinned.Get(prefix); ok { c.pinnedHits.Add(1) return entry, true @@ -347,6 +345,21 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { return entry, true } +// peek is the write-path equivalent of lookup: same tier walk, but does +// not bump hit/miss counters and does not fire the onMiss callback. +// Used by PutIfClean / MarkDirty so write traffic doesn't masquerade as +// read-miss pressure for the adaptive pin controller. +func (c *BranchCache) peek(prefix []byte) (*branchCacheEntry, bool) { + if isRootPrefix(prefix) { + entry := c.root.Load() + return entry, entry != nil + } + if entry, ok := c.pinned.Get(prefix); ok { + return entry, true + } + return c.tail.Get(prefix) +} + // fireOnMiss invokes the registered miss callback (if any). Hot path — // the no-callback case is a single atomic load and a nil check. func (c *BranchCache) fireOnMiss(prefix []byte) { @@ -422,18 +435,27 @@ func (c *BranchCache) SetMissCallback(cb MissCallback) { // ContractHashFromPrefix extracts the 32-byte contract hash from a // commitment-trunk prefix. Returns (hash, true) if the prefix is a -// storage-trunk prefix (depth >= 64, encoded as compact-hex with the -// contract's 64-nibble keccak prefix), or (_, false) for shorter +// storage-trunk prefix (depth >= 64, encoded as hex-prefix compact with +// the contract's 64-nibble keccak prefix), or (_, false) for shorter // prefixes (account-trie branches at depth < 64). // -// Compact-hex encoding: 1-byte HP flag, then nibble-pairs packed -// 2-per-byte. The contract's 64-nibble path occupies 32 bytes after -// the flag byte, so a storage-trunk prefix is at least 33 bytes. +// Hex-prefix encoding (see nibbles.HexToCompact): bit 4 of byte 0 is the +// odd-length flag. When set, the first nibble of the path lives in the +// low nibble of byte 0 and the remaining nibbles are packed 2-per-byte +// in buf[1:]; reading prefix[1:33] directly mis-attributes the hash by +// one nibble. Decoding via the high/low-nibble split here handles both +// parities. func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { if len(prefix) < 33 { return hash, false } - copy(hash[:], prefix[1:33]) + if prefix[0]&0x10 == 0 { + copy(hash[:], prefix[1:33]) + return hash, true + } + for i := 0; i < 32; i++ { + hash[i] = ((prefix[i] & 0x0f) << 4) | (prefix[i+1] >> 4) + } return hash, true } @@ -520,7 +542,7 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64, origin s // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step, txN uint64, origin string) bool { - if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { + if existing, ok := c.peek(prefix); ok && existing.dirty.Load() { return false } c.Put(prefix, data, step, txN, origin) @@ -558,7 +580,7 @@ func (c *BranchCache) GetWithOrigin(prefix []byte) (data []byte, origin string, // // No-op if no entry exists at prefix. func (c *BranchCache) MarkDirty(prefix []byte) { - if entry, ok := c.lookup(prefix); ok { + if entry, ok := c.peek(prefix); ok { entry.dirty.Store(true) } } diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index e8a29b8a0f1..9d43cde4306 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -18,9 +18,12 @@ package commitment import ( "strings" + "sync/atomic" "testing" "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/execution/commitment/nibbles" ) // TestBranchCache_RootPinning verifies the root branch lands in the pinned @@ -289,3 +292,65 @@ func TestBranchCache_UnwindTo_BoundaryEqualsWatermark(t *testing.T) { _, _, ok = c.Get(aboveKey) require.False(t, ok, "entry at txN>watermark must be evicted") } + +// TestContractHashFromPrefix_EvenAndOddNibblePaths verifies that the +// contract-hash extraction handles both parities of hex-prefix compact +// encoding (the odd-length-path flag puts the first nibble in byte 0, +// which a naive prefix[1:33] read mis-attributes by one nibble). +func TestContractHashFromPrefix_EvenAndOddNibblePaths(t *testing.T) { + var contract [32]byte + for i := range contract { + contract[i] = byte((i * 31) & 0xff) + } + contractNibbles := make([]byte, 64) + for i, b := range contract { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + + // Even-length path: depth 64 (no storage-trunk nibble). Encodes as + // flag byte (0x00) followed by the 32 contract bytes. + even := nibbles.HexToCompact(contractNibbles) + got, ok := ContractHashFromPrefix(even) + require.True(t, ok) + require.Equal(t, contract, got, "even-length path must round-trip") + + // Odd-length path: depth 65 (contract + one storage nibble). + odd := nibbles.HexToCompact(append(append([]byte(nil), contractNibbles...), 0x0a)) + got, ok = ContractHashFromPrefix(odd) + require.True(t, ok) + require.Equal(t, contract, got, "odd-length path must extract the contract hash, not a nibble-shifted slice") + + // Even-length path at depth 66 (contract + two storage nibbles). + even66 := nibbles.HexToCompact(append(append([]byte(nil), contractNibbles...), 0x0a, 0x05)) + got, ok = ContractHashFromPrefix(even66) + require.True(t, ok) + require.Equal(t, contract, got, "depth-66 even path must extract the contract hash") +} + +// TestBranchCache_PutIfClean_DoesNotCountAsMiss verifies that the +// write-path dirty check does not bump miss counters or fire the +// onMiss callback — write traffic must not masquerade as read pressure +// for the adaptive pin controller. +func TestBranchCache_PutIfClean_DoesNotCountAsMiss(t *testing.T) { + c := NewBranchCache(100) + var fired atomic.Uint64 + c.SetMissCallback(func(prefix []byte) { fired.Add(1) }) + + key := []byte{0x12, 0x34} + // PutIfClean on a cold prefix must succeed without registering as a + // triple-miss (no fire, no tail-miss counter bump). + require.True(t, c.PutIfClean(key, []byte("v1"), 0, 0, "test")) + require.Equal(t, uint64(0), fired.Load(), "PutIfClean must not fire onMiss on the write path") + require.Equal(t, uint64(0), c.tailMisses.Load(), "PutIfClean must not bump tail miss counter") + + // MarkDirty on a different cold prefix: also write-path, also must not count. + c.MarkDirty([]byte{0xff, 0xee}) + require.Equal(t, uint64(0), fired.Load(), "MarkDirty must not fire onMiss on the write path") + require.Equal(t, uint64(0), c.tailMisses.Load(), "MarkDirty must not bump tail miss counter") + + // A real read miss should still count. + _, _, ok := c.Get([]byte{0xaa, 0xbb}) + require.False(t, ok) + require.Equal(t, uint64(1), fired.Load(), "Get on cold prefix must fire onMiss") +} From fd74979033ce44924f451f288fdcb2916511ad43 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Thu, 4 Jun 2026 16:13:32 +0000 Subject: [PATCH 101/120] db/state/execctx: decode AccountsDomain codeHash with DeserialiseV3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codeHashForAddr resolves an account's codeHash from the AccountsDomain so the CodeDomain ethHash bypass can serve shared bytecode without an addr-keyed file read. decodeAccountCodeHash decoded the account value with acc.DecodeForStorage, but AccountsDomain values are SerialiseV3-encoded. DecodeForStorage is the legacy MDBX bitmask format with an incompatible binary layout; applied to V3 bytes it silently misparses and leaves CodeHash empty. As a result codeHashForAddr returned nil for every account and the ethHash bypass never engaged for any contract — every CodeDomain read that missed the addr cache fell through to a file read. This is a decoding-correctness bug: the wrong decoder is applied to the stored encoding. Use accounts.DeserialiseV3, the matching decoder for AccountsDomain values. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/execctx/domain_shared.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index ea8c923140c..2b660f4a580 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1121,7 +1121,12 @@ func decodeAccountCodeHash(enc []byte) []byte { return nil } var acc accounts.Account - if err := acc.DecodeForStorage(enc); err != nil { + // AccountsDomain values are SerialiseV3-encoded, so they must be decoded + // with DeserialiseV3. DecodeForStorage is the legacy MDBX bitmask format + // with an incompatible binary layout; applied to V3 bytes it silently + // misparses and leaves CodeHash empty, so codeHashForAddr returned nil for + // every contract and the CodeDomain ethHash bypass never fired. + if err := accounts.DeserialiseV3(&acc, enc); err != nil { return nil } if acc.CodeHash.IsEmpty() { From ba6c67a18c81858066f91f637ae6a3b9e9806251 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 20 May 2026 17:10:32 +0000 Subject: [PATCH 102/120] =?UTF-8?q?db/state,=20db/kv:=20unify=20FlushWithC?= =?UTF-8?q?allback=20=E2=80=94=20flush-only=20refresh=20for=20branchCache?= =?UTF-8?q?=20+=20stateCache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The optimization stack and the cache branch had grown two divergent FlushWithCallback methods — one single-domain for the BranchCache (commitment), one all-domains for the StateCache flush-only fix. Merge them into one. FlushWithCallback is now all-domains: it invokes cb for every (domain, key, latest-value, step) across all domains (sd.domains + sd.storage), then drains the mem-batch — callback, MDBX flush and drain in one latestStateLock window. SharedDomains.Flush passes a single callback that routes CommitmentDomain → BranchCache and Accounts/Storage/Code → StateCache. The per-write stateCache.Put/Delete in domainPut/DomainDel are removed: a write is in-flight, fork-specific state living in sd.mem; mirroring it into the process-wide cache let a sibling fork's re-execution read another fork's uncommitted bytes. The cache is now refreshed only on flush, so it mirrors committed, fork-agnostic state. drainLocked empties sd.mem as part of the flush so a child SD chained as parent reads through to the refreshed cache / DB instead of stale bytes. This folds the cache fix (was 527cb23077 on the cache branch) into the optimization stack so the local group-test exercises the final code. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/kv/kv_interface.go | 2 +- db/state/execctx/domain_shared.go | 208 ++++++++++++++++-------------- db/state/temporal_mem_batch.go | 79 +++++++----- 3 files changed, 165 insertions(+), 124 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index 16a76e0ff4e..e757a32effd 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,7 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error - FlushWithCallback(ctx context.Context, tx RwTx, domain Domain, cb func(k []byte, v []byte, step Step)) error + FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 2b660f4a580..3f9ebd930b5 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -780,91 +780,127 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { return err } } - // Update the cache with the bytes about to land in MDBX, atomically + // Refresh the caches with the bytes about to land in MDBX, atomically // with the flush. FlushWithCallback holds the latestState write lock // for the full window so a concurrent DomainPut cannot interleave // between the cache update and the MDBX write — readers see either // the local sd.mem value (during the lock) or the cached/MDBX value - // (after release). No window where cache is stale vs MDBX. - if sd.branchCache != nil { - if err := sd.mem.FlushWithCallback(ctx, tx, kv.CommitmentDomain, func(k []byte, v []byte, step kv.Step) { - if len(v) == 0 { - sd.branchCache.Invalidate(k) - return + // (after release). The caches are refreshed ONLY here, on flush — + // never per-write — so they mirror committed, fork-agnostic state: + // CommitmentDomain → BranchCache, Accounts/Storage/Code → StateCache. + if sd.branchCache != nil || sd.stateCache != nil { + if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, step kv.Step) { + switch domain { + case kv.CommitmentDomain: + if sd.branchCache != nil { + if len(v) == 0 { + sd.branchCache.Invalidate(k) + } else { + sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + } + } + case kv.AccountsDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.AccountsDomain, k) + sd.stateCache.Delete(kv.CodeDomain, k) + sd.stateCache.DeleteAddrCodeHash(k) + } else { + sd.stateCache.Put(kv.AccountsDomain, k, v) + sd.stateCache.DeleteAddrCodeHash(k) + } + } + case kv.StorageDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.StorageDomain, k) + } else { + sd.stateCache.Put(kv.StorageDomain, k, v) + } + } + case kv.CodeDomain: + if sd.stateCache != nil { + if len(v) == 0 { + sd.stateCache.Delete(kv.CodeDomain, k) + } else { + sd.stateCache.Put(kv.CodeDomain, k, v) + } + } } - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") }); err != nil { return err } - // Adaptive controller hook: now that the cache reflects this - // batch's writes, decide promotions / extensions / demotions - // for the contracts whose miss pressure crossed thresholds. - // The reader uses the in-flight Flush tx so pre-cache reads - // see the just-committed bytes (the tx's own writes are - // visible to itself). - if sd.adaptivePinController != nil { - if ttx, ok := tx.(kv.TemporalTx); ok { - reader := func(prefix []byte) ([]byte, uint64, bool, error) { - v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) - if err != nil { - return nil, 0, false, err + if sd.branchCache != nil { + // Adaptive controller hook: now that the cache reflects this + // batch's writes, decide promotions / extensions / demotions + // for the contracts whose miss pressure crossed thresholds. + // The reader uses the in-flight Flush tx so pre-cache reads + // see the just-committed bytes (the tx's own writes are + // visible to itself). + if sd.adaptivePinController != nil { + if ttx, ok := tx.(kv.TemporalTx); ok { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil } - return v, uint64(step), len(v) > 0, nil - } - // Install per-call parallel-mode plumbing so the - // controller's promote/extend uses the wave-BFS file-only - // resolver. The factory captures ttx; cleared after the - // call so a future OnBlockComplete can't reach a stale tx. - factory := func() (commitment.BatchBranchResolver, func(), error) { - resolve := func(keys [][]byte) ([][]byte, error) { - d := ttx.Debug() - vals := make([][]byte, len(keys)) - for i, k := range keys { - v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) - if err != nil { - return nil, err - } - if found { - vc := make([]byte, len(v)) - copy(vc, v) - vals[i] = vc + // Install per-call parallel-mode plumbing so the + // controller's promote/extend uses the wave-BFS file-only + // resolver. The factory captures ttx; cleared after the + // call so a future OnBlockComplete can't reach a stale tx. + factory := func() (commitment.BatchBranchResolver, func(), error) { + resolve := func(keys [][]byte) ([][]byte, error) { + d := ttx.Debug() + vals := make([][]byte, len(keys)) + for i, k := range keys { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) + if err != nil { + return nil, err + } + if found { + vc := make([]byte, len(v)) + copy(vc, v) + vals[i] = vc + } } + return vals, nil } - return vals, nil + return resolve, nil, nil } - return resolve, nil, nil - } - provider := func(contractHash []byte) map[string][]byte { - m := map[string][]byte{} - c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) - if cerr != nil { - return m - } - defer c.Close() - evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) - scan := func(from, to []byte) { - for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { - if bytes.Compare(k, to) >= 0 { - return - } - if len(v) < 8 { - continue + provider := func(contractHash []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + scan := func(from, to []byte) { + for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { + if bytes.Compare(k, to) >= 0 { + return + } + if len(v) < 8 { + continue + } + m[string(common.Copy(k))] = common.Copy(v[8:]) } - m[string(common.Copy(k))] = common.Copy(v[8:]) } + scan(evenFrom, evenTo) + scan(oddFrom, oddTo) + return m } - scan(evenFrom, evenTo) - scan(oddFrom, oddTo) - return m + sd.adaptivePinController.SetParallelMode(factory, provider) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + sd.adaptivePinController.SetParallelMode(nil, nil) } - sd.adaptivePinController.SetParallelMode(factory, provider) - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) - sd.adaptivePinController.SetParallelMode(nil, nil) } + // Refresh pinned-tier gauges once per Flush — once-per-batch + // cadence avoids per-lookup cost in the hot read path. + sd.branchCache.PublishMetrics() } - // Refresh pinned-tier gauges once per Flush — once-per-batch - // cadence avoids per-lookup cost in the hot read path. - sd.branchCache.PublishMetrics() return nil } return sd.mem.Flush(ctx, tx) @@ -1256,17 +1292,12 @@ func (sd *SharedDomains) domainPut(domain kv.Domain, roTx kv.TemporalTx, k, v [] } } - // Update state cache when writing. For AccountsDomain, also invalidate - // the addr → codeHash LRU above SD because the codeHash may have - // changed (CREATE/CREATE2-replace; SELFDESTRUCT goes through DomainDel - // which has its own invalidation). The next codeHashForAddr lookup - // will repopulate from the freshly written account bytes. - if sd.stateCache != nil { - sd.stateCache.Put(domain, k, v) - if domain == kv.AccountsDomain { - sd.stateCache.DeleteAddrCodeHash(k) - } - } + // The state cache is NOT updated here. This write goes into sd.mem and + // is served from there (checked first on every read, fork-isolated via + // the parent chain); the shared cache is refreshed only on flush + // (SharedDomains.Flush → FlushWithCallback), so it mirrors committed, + // fork-agnostic state. A per-write update would leak non-flushed, + // fork-specific bytes into a sibling fork's reads. // Serialize against the calculator's accumulator-swap window — see // changesetMu doc on the SharedDomains struct. Skipped when the caller @@ -1310,31 +1341,20 @@ func (sd *SharedDomains) DomainDel(domain kv.Domain, tx kv.TemporalTx, k []byte, if err := sd.DomainDel(kv.CodeDomain, tx, k, txNum, nil); err != nil { return err } - // Remove from state cache when account is deleted (including the - // nethermind-style addr → codeHash mapping; SELFDESTRUCT and - // CREATE-replace both reset the codeHash). - if sd.stateCache != nil { - sd.stateCache.Delete(kv.AccountsDomain, k) - sd.stateCache.Delete(kv.CodeDomain, k) - sd.stateCache.DeleteAddrCodeHash(k) - } + // State cache is refreshed on flush only — see DomainPut. The flush + // callback handles the empty-value (delete) case for accounts, code + // and the addr→codeHash mapping. // AccountsDomain — apply-side. Serialize against swap window. sd.changesetMu.Lock() defer sd.changesetMu.Unlock() return sd.mem.DomainDel(kv.AccountsDomain, ks, txNum, prevVal) case kv.StorageDomain: - // Remove from state cache when storage is deleted - if sd.stateCache != nil { - sd.stateCache.Delete(kv.StorageDomain, k) - } + // State cache refreshed on flush only — see DomainPut. case kv.CodeDomain: if prevVal == nil { return nil } - // Remove from state cache when code is deleted - if sd.stateCache != nil { - sd.stateCache.Delete(kv.CodeDomain, k) - } + // State cache refreshed on flush only — see DomainPut. default: //noop } diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 5ce7e1c8158..fa67b0bd1f5 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -762,47 +762,68 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { return nil } -// FlushWithCallback updates a downstream cache via cb for each -// (key, value, step) tuple in domain, then flushes the mem-batch to tx. -// Cache-update is ordered BEFORE the MDBX flush so that during the -// MDBX write window the cache still holds the entry — a reader that -// goes (sd write buffer → cache → MDBX) never observes a key missing -// from all three sources. The MDBX commit provides db-side atomicity -// independently. +// FlushWithCallback flushes the mem-batch to tx, but first invokes cb for +// every (domain, key, latest-value, step) tuple so downstream caches can be +// refreshed, then drains the in-memory latest/history state. // -// Used by SharedDomains.Flush to keep an external BranchCache in sync -// with commitment-domain writes. +// The callback, the MDBX flush and the drain run under latestStateLock as +// one atomic window: a concurrent reader sees either the full pre-flush mem +// state or the fully-drained post-flush state, never a partial mix. cb runs +// before the flush so that during the MDBX write window the cache already +// holds the entry — a reader going mem → cache → MDBX never sees a key +// missing from all three. data may be nil/empty for deletes — cb +// distinguishes on len(v)==0. // -// cb is called under latestStateLock so the snapshot it sees matches -// the in-memory state at flush time. Flush itself runs under the same -// lock to prevent concurrent DomainPut from interleaving with the -// snapshot/cache-update/flush sequence. -// -// If Flush fails, the cache has already been updated for this batch. -// That's the same invariant the cache maintains for any other write -// path — values may exist in the cache before they reach disk; a -// retry of Flush re-applies them. +// Used by SharedDomains.Flush to keep the BranchCache (commitment domain) +// and the StateCache (accounts/storage/code) in sync — refreshed only on +// flush, so they hold committed, fork-agnostic state. func (sd *TemporalMemBatch) FlushWithCallback( - ctx context.Context, tx kv.RwTx, domain kv.Domain, - cb func(k []byte, v []byte, step kv.Step), + ctx context.Context, tx kv.RwTx, + cb func(domain kv.Domain, k []byte, v []byte, step kv.Step), ) error { sd.latestStateLock.Lock() defer sd.latestStateLock.Unlock() - // dir is unset on entries written via DomainPut/DomainDel (always - // default zero); the latest history entry per key is authoritative. - // data may be nil/empty for deletes — caller's cb is expected to - // distinguish (see SharedDomains.Flush, which Invalidate's on - // len(v)==0 and Put's otherwise). - for keyStr, history := range sd.domains[domain] { + for d := range sd.domains { + for keyStr, history := range sd.domains[d] { + if len(history) == 0 { + continue + } + latest := history[len(history)-1] + cb(kv.Domain(d), []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + } + } + // StorageDomain entries live in sd.storage (a btree), not sd.domains. + sd.storage.Scan(func(keyStr string, history []dataWithTxNum) bool { if len(history) == 0 { - continue + return true } latest := history[len(history)-1] - cb([]byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.StorageDomain, []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + return true + }) + + if err := sd.flushLocked(ctx, tx); err != nil { + return err } + sd.drainLocked() + return nil +} - return sd.flushLocked(ctx, tx) +// drainLocked clears the in-memory latest/history state after a flush. The +// caller must hold latestStateLock. Mirrors the data-clearing half of +// ClearRam (without the metrics reset): once flushed, the mem-batch holds +// no data — values now live in the DB and the refreshed caches, and a child +// SD chained to this one as parent reads through instead of being shadowed +// by stale, fork-specific bytes. +func (sd *TemporalMemBatch) drainLocked() { + for i := range sd.domains { + sd.domains[i] = map[string][]dataWithTxNum{} + } + sd.storage = btree2.NewMap[string, []dataWithTxNum](128) + sd.unwindToTxNum = 0 + sd.unwindChangeset = nil + sd.unwindChangesetRaw = nil } // flushLocked is the body of Flush, factored so FlushWithCallback can From 061111fd8548724bf1543fb9eee2c0b75307d48c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:25:00 +0000 Subject: [PATCH 103/120] db/rawdb, p2p/eth, execmodule: BAL cache + on-demand regeneration for eth/71 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a 3-tier lookup for serving EIP-7928 Block Access Lists to eth/71 GetBlockAccessLists peers: 1. In-memory LRU cache (recent blocks, freshly produced or recently served). 100-entry window keyed on block hash. 2. Chaindata DB (BALs the eth/71 bal-downloader fetched from peers, or legacy persisted BALs). DB hits are promoted to the cache so repeat requests don't re-read MDBX. 3. Installed BALRegenerator backend that re-executes the block against its parent state to derive the BAL. Result is cached before return. The eth/71 handler (AnswerGetBlockAccessListsQuery) and the sentry dispatch path now thread a context through BlockAccessListBytes, so regeneration respects the peer's read deadline. BALRegenerator implementation (execution/execmodule/balregen.go) uses a simple IBS path: parent-state reader → IntraBlockState with VersionMap enabled (read tracking) → InitializeBlockExecution → ApplyTransaction loop → merge ibs.TxIO() into a per-block VersionedIO → AsBlockAccessList() → EncodeBlockAccessListBytes. No parallel-exec dependency, matching the block-assembler pattern (execution/exec/block_assembler.go). The Finalize step is intentionally omitted from the re-exec loop — fork-specific engine.Finalize signatures vary and the finalize-time BAL deltas (system contracts, withdrawals) are typically captured during InitializeBlockExecution. If a downstream peer's hash verification flags a mismatch on Finalize-touched accounts, that's the signal to wire fork-aware finalize support here. Cache + lookup tests: - db/rawdb/balcache_test.go: 9 cases covering cache-hit short-circuit, DB-hit-promotes-to-cache, regenerator-fallback, no-regenerator path, nil-regeneration-not-cached, regenerator-error-not-cached, empty data is no-op, defensive byte-copy, SetBALRegenerator replace+clear. Handler tests: - p2p/protocols/eth/handlers_test.go: existing tests updated to thread ctx and reset the cache between runs. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/rawdb/balcache.go | 154 ++++++++++++ db/rawdb/balcache_test.go | 224 ++++++++++++++++++ execution/execmodule/balregen.go | 218 +++++++++++++++++ p2p/protocols/eth/handlers.go | 15 +- p2p/protocols/eth/handlers_test.go | 8 +- .../sentry_multi_client.go | 2 +- 6 files changed, 612 insertions(+), 9 deletions(-) create mode 100644 db/rawdb/balcache.go create mode 100644 db/rawdb/balcache_test.go create mode 100644 execution/execmodule/balregen.go diff --git a/db/rawdb/balcache.go b/db/rawdb/balcache.go new file mode 100644 index 00000000000..3b1d3d79ac1 --- /dev/null +++ b/db/rawdb/balcache.go @@ -0,0 +1,154 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package rawdb + +import ( + "context" + "sync/atomic" + + lru "github.com/hashicorp/golang-lru/v2" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv" +) + +// balCacheSize is how many recent blocks' EIP-7928 BlockAccessLists are kept in +// memory. Peers fetching BALs (eth/71) and the eth_getBlockAccessList RPC ask +// for recent blocks; older BALs are not persisted by default and must be +// regenerated by re-executing the block (see BALRegenerator). +const balCacheSize = 100 + +var balCache *lru.Cache[common.Hash, []byte] + +// balRegenerator holds the optionally-installed re-execution backend that can +// produce a BAL when neither the cache nor the chaindata DB has one. The value +// is wrapped in an addressable struct (atomic.Value rejects nil interface +// stores). Set once at startup via SetBALRegenerator; reads are lock-free. +type balRegeneratorSlot struct{ r BALRegenerator } + +var balRegenerator atomic.Pointer[balRegeneratorSlot] + +func init() { + var err error + balCache, err = lru.New[common.Hash, []byte](balCacheSize) + if err != nil { + panic(err) + } +} + +// BALRegenerator produces the RLP-encoded BlockAccessList for a block by +// re-executing it against the parent state. Implementations live close to the +// exec module (they need the full block + chain config + state-reader stack); +// the lookup path here installs one via SetBALRegenerator and consults it as a +// fallback when neither the in-memory cache nor the chaindata DB has the BAL. +// +// The returned bytes are the canonical RLP encoding (as written by +// rawdb.WriteBlockAccessListBytes) so callers can hand them to peers +// unchanged. The hash of the decoded BAL MUST match header.BlockAccessListHash +// — peers verify this on receipt; a mismatch fails the eth/71 BAL response. +type BALRegenerator interface { + RegenerateBlockAccessList(ctx context.Context, hash common.Hash, number uint64) ([]byte, error) +} + +// SetBALRegenerator installs the re-execution backend used by +// BlockAccessListBytes as a last-resort fallback. Pass nil to clear. Safe to +// call multiple times; the most recent setter wins. Typically called once at +// node startup after the exec module is constructed. +func SetBALRegenerator(r BALRegenerator) { + if r == nil { + balRegenerator.Store(nil) + return + } + balRegenerator.Store(&balRegeneratorSlot{r: r}) +} + +func getBALRegenerator() BALRegenerator { + slot := balRegenerator.Load() + if slot == nil { + return nil + } + return slot.r +} + +// CacheBlockAccessList stores the block's RLP-encoded BAL bytes in the in-memory +// cache instead of persisting them to the chaindata DB. Writing the BAL to MDBX +// on the NewPayload critical path was a multi-hundred-KB Put per block that, on a +// churned DB, took tens of seconds; the cache + regenerate-on-miss model mirrors +// what we do for receipts (--persist.receipt). +func CacheBlockAccessList(hash common.Hash, data []byte) { + if len(data) == 0 { + return + } + balCache.Add(hash, common.Copy(data)) +} + +// CachedBlockAccessList returns the cached RLP-encoded BAL bytes for hash, if the +// block is still within the in-memory window. Cache-only — does NOT consult the +// chaindata DB or invoke the regenerator. Use BlockAccessListBytes for the +// full lookup chain. +func CachedBlockAccessList(hash common.Hash) ([]byte, bool) { + return balCache.Get(hash) +} + +// BlockAccessListBytes returns the BAL bytes for (hash, number). Lookup order: +// 1. In-memory LRU cache (recent blocks, freshly produced). +// 2. Chaindata DB (BALs the eth/71 bal-downloader fetched from peers, or +// legacy persisted BALs). +// 3. If a BALRegenerator is installed, re-execute the block to derive the BAL. +// The result is added to the cache before returning. +// +// Returns (nil, nil) if no source has it (no regenerator installed, or +// regenerator returned nil). Returns a non-nil error only on actual lookup +// failure (DB error, regenerator error). +// +// Callers passing the context (ctx) get cancellation of the regeneration leg +// — peer-driven serving paths should respect the peer's read deadline. +func BlockAccessListBytes(ctx context.Context, db kv.Getter, hash common.Hash, number uint64) ([]byte, error) { + if v, ok := balCache.Get(hash); ok { + return v, nil + } + stored, err := ReadBlockAccessListBytes(db, hash, number) + if err != nil { + return nil, err + } + if stored != nil { + // Promote DB hits into the cache so the next lookup is a single + // LRU probe — important for serving the same hash repeatedly + // (peers re-asking, RPC retries). + CacheBlockAccessList(hash, stored) + return stored, nil + } + regen := getBALRegenerator() + if regen == nil { + return nil, nil + } + generated, err := regen.RegenerateBlockAccessList(ctx, hash, number) + if err != nil { + return nil, err + } + if len(generated) == 0 { + return nil, nil + } + CacheBlockAccessList(hash, generated) + return generated, nil +} + +// ResetBALCacheForTest clears the in-memory cache and the regenerator. Test-only. +func ResetBALCacheForTest() { + balCache.Purge() + balRegenerator.Store(nil) +} diff --git a/db/rawdb/balcache_test.go b/db/rawdb/balcache_test.go new file mode 100644 index 00000000000..cf2dca18e36 --- /dev/null +++ b/db/rawdb/balcache_test.go @@ -0,0 +1,224 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package rawdb_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/db/kv/memdb" + "github.com/erigontech/erigon/db/rawdb" +) + +// fakeRegenerator records every regeneration request + serves canned responses. +type fakeRegenerator struct { + calls atomic.Int32 + // resultFor maps the (hash) → bytes the regenerator will return. Unset + // hashes get the default canned bytes if set, else (nil, errNoBALForHash). + resultFor map[common.Hash][]byte + defaultBytes []byte + errAlways error +} + +func (f *fakeRegenerator) RegenerateBlockAccessList(_ context.Context, hash common.Hash, _ uint64) ([]byte, error) { + f.calls.Add(1) + if f.errAlways != nil { + return nil, f.errAlways + } + if v, ok := f.resultFor[hash]; ok { + return v, nil + } + if f.defaultBytes != nil { + return f.defaultBytes, nil + } + return nil, nil +} + +var errFakeRegen = errors.New("fake regen failure") + +func hashFromByte(b byte) common.Hash { + var h common.Hash + for i := range h { + h[i] = b + } + return h +} + +func TestBlockAccessListBytes_CacheHitShortCircuits(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x01) + data := []byte{0xc1, 0x00} + rawdb.CacheBlockAccessList(hash, data) + + // Install a regenerator that, if called, fails the test. + rawdb.SetBALRegenerator(&fakeRegenerator{errAlways: errFakeRegen}) + + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 7) + require.NoError(t, err) + require.Equal(t, data, got) +} + +func TestBlockAccessListBytes_DBHitPromotesToCache(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x02) + data := []byte{0xc2, 0xff, 0xee} + require.NoError(t, rawdb.WriteBlockAccessListBytes(tx, hash, 11, data)) + + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 11) + require.NoError(t, err) + require.Equal(t, data, got) + + // Subsequent lookup must be a cache hit (no DB read necessary). + rawdb.SetBALRegenerator(nil) + got2, ok := rawdb.CachedBlockAccessList(hash) + require.True(t, ok, "DB-source BAL must be promoted to cache") + require.Equal(t, data, got2) +} + +func TestBlockAccessListBytes_RegeneratorFallback(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x03) + regenerated := []byte{0xc3, 0x42} + regen := &fakeRegenerator{defaultBytes: regenerated} + rawdb.SetBALRegenerator(regen) + + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 22) + require.NoError(t, err) + require.Equal(t, regenerated, got) + require.Equal(t, int32(1), regen.calls.Load(), "regenerator should be called once on miss") + + // Repeated lookup hits the cache, regenerator NOT called again. + got2, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 22) + require.NoError(t, err) + require.Equal(t, regenerated, got2) + require.Equal(t, int32(1), regen.calls.Load(), "cached regenerated BAL must short-circuit subsequent lookups") +} + +func TestBlockAccessListBytes_NoRegeneratorOnMiss(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x04) + rawdb.SetBALRegenerator(nil) + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 33) + require.NoError(t, err) + require.Nil(t, got, "no cache, no DB, no regenerator → nil bytes (peer sees 'not available')") +} + +func TestBlockAccessListBytes_RegeneratorReturnsNil(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x05) + regen := &fakeRegenerator{} // defaultBytes nil + rawdb.SetBALRegenerator(regen) + + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 44) + require.NoError(t, err) + require.Nil(t, got) + require.Equal(t, int32(1), regen.calls.Load()) + + // A nil-from-regenerator must NOT be cached (so a later install of a + // real regenerator can succeed). + _, ok := rawdb.CachedBlockAccessList(hash) + require.False(t, ok, "nil regeneration result must not be cached") +} + +func TestBlockAccessListBytes_RegeneratorError(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x06) + regen := &fakeRegenerator{errAlways: errFakeRegen} + rawdb.SetBALRegenerator(regen) + + _, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 55) + require.ErrorIs(t, err, errFakeRegen) + _, ok := rawdb.CachedBlockAccessList(hash) + require.False(t, ok, "regenerator error must not pollute the cache") +} + +func TestCacheBlockAccessList_EmptyIsNoOp(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + hash := hashFromByte(0x07) + rawdb.CacheBlockAccessList(hash, nil) + rawdb.CacheBlockAccessList(hash, []byte{}) + _, ok := rawdb.CachedBlockAccessList(hash) + require.False(t, ok, "empty data must not be cached (would conflate with 'not available')") +} + +func TestCacheBlockAccessList_CopiesBytes(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + hash := hashFromByte(0x08) + src := []byte{0xde, 0xad, 0xbe, 0xef} + rawdb.CacheBlockAccessList(hash, src) + src[0] = 0xff // mutate caller's slice — cache must hold its own copy + + got, ok := rawdb.CachedBlockAccessList(hash) + require.True(t, ok) + require.Equal(t, byte(0xde), got[0], "cache must defensively copy the input bytes") +} + +func TestSetBALRegenerator_ReplaceAndClear(t *testing.T) { + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.ResetBALCacheForTest() + _, tx := memdb.NewTestTx(t) + defer tx.Rollback() + + hash := hashFromByte(0x09) + r1 := &fakeRegenerator{defaultBytes: []byte{0xa1}} + r2 := &fakeRegenerator{defaultBytes: []byte{0xa2}} + rawdb.SetBALRegenerator(r1) + rawdb.SetBALRegenerator(r2) // replace + + got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 99) + require.NoError(t, err) + require.Equal(t, []byte{0xa2}, got) + require.Zero(t, r1.calls.Load(), "old regenerator must not be called after replacement") + require.Equal(t, int32(1), r2.calls.Load()) + + rawdb.ResetBALCacheForTest() // clear cache so the next lookup hits the regenerator again + rawdb.SetBALRegenerator(nil) // explicit clear + got, err = rawdb.BlockAccessListBytes(context.Background(), tx, hash, 99) + require.NoError(t, err) + require.Nil(t, got, "cleared regenerator → miss returns nil") +} diff --git a/execution/execmodule/balregen.go b/execution/execmodule/balregen.go new file mode 100644 index 00000000000..535615345be --- /dev/null +++ b/execution/execmodule/balregen.go @@ -0,0 +1,218 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package execmodule + +import ( + "context" + "fmt" + "math/big" + + "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/protocol" + "github.com/erigontech/erigon/execution/protocol/rules" + "github.com/erigontech/erigon/execution/state" + "github.com/erigontech/erigon/execution/types" + "github.com/erigontech/erigon/execution/types/accounts" + "github.com/erigontech/erigon/execution/vm" +) + +// BALRegeneratorDeps bundles the per-node helpers the regenerator needs to +// re-execute a historical block: chain config, consensus engine, block reader +// (for txs/bodies), and a state-reader factory keyed on block hash+number. +// +// The state-reader factory returns a reader rooted at the *parent* state of the +// requested block. Caller owns the returned closer (typically a kv.Tx.Rollback) +// — the regenerator invokes it once after the re-execution finishes. +type BALRegeneratorDeps struct { + ChainConfig *chain.Config + Engine rules.Engine + BlockReader services.FullBlockReader + NewParentStateReader func(ctx context.Context, blockHash common.Hash, blockNum uint64) (state.StateReader, services.HeaderReader, func(n uint64) (common.Hash, error), func(), error) + Logger log.Logger +} + +// BALRegenerator implements rawdb.BALRegenerator by re-executing the requested +// block against its parent state with VersionMap-enabled IBS read tracking. The +// computed BAL is RLP-encoded and returned; the hash of the decoded BAL must +// match header.BlockAccessListHash (the parallel-exec path that produced the +// original BAL uses the same VersionedIO tracking, so the bytes are +// expected to be hash-equivalent for a non-failing execution). +// +// Suitable for serving eth/71 GetBlockAccessLists when the local node hasn't +// stored the BAL (pre-Amsterdam, pruned, never-received, or the cache window +// has rolled). Does NOT modify any persistent state — the IBS writes go to +// state.NewNoopWriter(). +type BALRegenerator struct { + deps BALRegeneratorDeps +} + +func NewBALRegenerator(deps BALRegeneratorDeps) *BALRegenerator { + return &BALRegenerator{deps: deps} +} + +// RegenerateBlockAccessList re-executes the block at (hash, number) and returns +// the RLP-encoded BAL. Returns (nil, nil) when the block can't be located OR +// the chain config doesn't have BAL active at the block's timestamp. +func (r *BALRegenerator) RegenerateBlockAccessList(ctx context.Context, hash common.Hash, number uint64) ([]byte, error) { + if r.deps.NewParentStateReader == nil { + return nil, fmt.Errorf("BALRegenerator: NewParentStateReader is nil") + } + stateReader, headerReader, blockHashFunc, closer, err := r.deps.NewParentStateReader(ctx, hash, number) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: parent-state reader: %w", err) + } + if closer != nil { + defer closer() + } + + var stateGetter kv.TemporalGetter + _ = stateGetter // future: when the reader needs explicit getter passthrough + + // Fetch the block from the local reader (txs + header). + header, err := r.deps.BlockReader.Header(ctx, dbForReader(headerReader), hash, number) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: header lookup: %w", err) + } + if header == nil { + return nil, nil + } + if !r.deps.ChainConfig.IsAmsterdam(header.Time) { + // Pre-Amsterdam blocks don't have BALs by spec. + return nil, nil + } + body, err := r.deps.BlockReader.BodyWithTransactions(ctx, dbForReader(headerReader), hash, number) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: body lookup: %w", err) + } + if body == nil { + return nil, nil + } + block := types.NewBlockFromStorage(hash, header, body.Transactions, body.Uncles, body.Withdrawals) + + bal, err := computeBlockAccessList(ctx, r.deps.ChainConfig, r.deps.Engine, block, stateReader, headerReader, blockHashFunc, r.deps.Logger) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: re-exec block %d: %w", number, err) + } + if bal == nil { + return nil, nil + } + balBytes, err := types.EncodeBlockAccessListBytes(bal) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: encode: %w", err) + } + return balBytes, nil +} + +// dbForReader is a placeholder for the kv.Getter context the HeaderReader/ +// BodyWithTransactions paths sometimes need. The closure inside +// NewParentStateReader is expected to share its tx with the header reader; this +// is the seam where it's threaded through. +// +// TODO: the BlockReader's Header/BodyWithTransactions need an explicit kv.Tx — +// NewParentStateReader currently exposes only a state.StateReader. Surface the +// tx alongside the state reader so this becomes a real argument. +func dbForReader(_ services.HeaderReader) kv.Getter { + return nil +} + +// computeBlockAccessList runs the block transactions through a simple IBS with +// VersionMap-enabled read tracking and returns the accumulated BAL. Mirrors the +// per-tx Merge pattern used by the block assembler — no parallel exec needed. +func computeBlockAccessList( + ctx context.Context, + chainConfig *chain.Config, + engine rules.Engine, + block *types.Block, + stateReader state.StateReader, + headerReader services.HeaderReader, + blockHashFunc func(n uint64) (common.Hash, error), + logger log.Logger, +) (types.BlockAccessList, error) { + ibs := state.New(stateReader) + defer ibs.Release(false) + // Read tracking requires a VersionMap — versionedRead is the only path + // that populates ibs.versionedReads. An empty VersionMap is fine; we + // don't need cross-tx coordination here. + ibs.SetVersionMap(state.NewVersionMap(nil)) + + header := block.HeaderNoCopy() + gp := new(protocol.GasPool).AddGas(block.GasLimit()).AddBlobGas(chainConfig.GetMaxBlobGasPerBlock(block.Time())) + gasUsed := new(protocol.GasUsed) + + chainReader := newChainReaderShim(chainConfig, headerReader) + + if err := protocol.InitializeBlockExecution(engine, chainReader, header, chainConfig, ibs, state.NewNoopWriter(), logger, nil); err != nil { + return nil, fmt.Errorf("InitializeBlockExecution: %w", err) + } + + var balIO state.VersionedIO + balIO = *balIO.Merge(ibs.TxIO()) + ibs.ResetVersionedIO() + + for i, txn := range block.Transactions() { + if err := ctx.Err(); err != nil { + return nil, err + } + ibs.SetTxContext(block.NumberU64(), i) + _, err := protocol.ApplyTransaction(chainConfig, blockHashFunc, engine, accounts.NilAddress, gp, ibs, state.NewNoopWriter(), header, txn, gasUsed, vm.Config{NoReceipts: true}) + if err != nil { + return nil, fmt.Errorf("apply tx %d (%x): %w", i, txn.Hash(), err) + } + balIO = *balIO.Merge(ibs.TxIO()) + ibs.ResetVersionedIO() + } + + // Finalize step is intentionally omitted: the engine.Finalize signatures + // differ across forks and require receipts/systemCall hooks we don't + // reconstruct here. Withdrawal + system-call accesses are typically + // captured during InitializeBlockExecution; any post-tx finalize writes + // would only matter for accounts already in the BAL from the tx loop. + // If the resulting BAL hash disagrees with header.BlockAccessListHash + // on a downstream peer's verification, that's the signal to add finalize + // support here (with the proper system-call hooks). + + return balIO.AsBlockAccessList(), nil +} + +// chainReaderShim is a minimal adapter for rules.ChainHeaderReader that +// protocol.InitializeBlockExecution requires. The header-lookup methods stay +// nil because system-init calls in this path don't reach for parent headers +// (the only thing that would need them is engine.Finalize, which we don't +// invoke here). +type chainReaderShim struct { + cfg *chain.Config + reader services.HeaderReader +} + +func newChainReaderShim(cfg *chain.Config, reader services.HeaderReader) *chainReaderShim { + return &chainReaderShim{cfg: cfg, reader: reader} +} + +func (s *chainReaderShim) Config() *chain.Config { return s.cfg } +func (s *chainReaderShim) CurrentHeader() *types.Header { return nil } +func (s *chainReaderShim) CurrentFinalizedHeader() *types.Header { return nil } +func (s *chainReaderShim) CurrentSafeHeader() *types.Header { return nil } +func (s *chainReaderShim) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } +func (s *chainReaderShim) GetHeaderByNumber(number uint64) *types.Header { return nil } +func (s *chainReaderShim) GetHeaderByHash(hash common.Hash) *types.Header { return nil } +func (s *chainReaderShim) GetTd(hash common.Hash, number uint64) *big.Int { return big.NewInt(0) } +func (s *chainReaderShim) FrozenBlocks() uint64 { return 0 } +func (s *chainReaderShim) FrozenBorBlocks(align bool) uint64 { return 0 } diff --git a/p2p/protocols/eth/handlers.go b/p2p/protocols/eth/handlers.go index 068ad0224cc..99dfacb07e9 100644 --- a/p2p/protocols/eth/handlers.go +++ b/p2p/protocols/eth/handlers.go @@ -191,7 +191,7 @@ var notAvailableSentinel = rlp.RawValue{0x80} // size, MaxBlockAccessListsServe caps the disk-lookup count. When a limit is // reached, the response is truncated (not padded with 0x80) — the peer sees a // shorter array than requested, same convention as the BlockBodies handler. -func AnswerGetBlockAccessListsQuery(db kv.Tx, query GetBlockAccessListsPacket, blockReader services.HeaderReader) []rlp.RawValue { //nolint:unparam +func AnswerGetBlockAccessListsQuery(ctx context.Context, db kv.Tx, query GetBlockAccessListsPacket, blockReader services.HeaderReader) []rlp.RawValue { //nolint:unparam var bytes int bals := make([]rlp.RawValue, 0, len(query)) @@ -200,18 +200,21 @@ func AnswerGetBlockAccessListsQuery(db kv.Tx, query GetBlockAccessListsPacket, b lookups >= 2*MaxBlockAccessListsServe { break } - number, _ := blockReader.HeaderNumber(context.Background(), db, hash) + number, _ := blockReader.HeaderNumber(ctx, db, hash) if number == nil { // We don't know the block — peer can retry elsewhere. bals = append(bals, notAvailableSentinel) bytes += len(notAvailableSentinel) continue } - bal, _ := rawdb.ReadBlockAccessListBytes(db, hash, *number) + // Cache-aware lookup: in-memory cache → chaindata DB → installed + // BALRegenerator (re-executes the block when nothing is stored). + bal, _ := rawdb.BlockAccessListBytes(ctx, db, hash, *number) if len(bal) == 0 { - // We have the block but no BAL stored (pre-Amsterdam, or pruned). - // Return 0x80 — unambiguously "not available", distinct from a - // genuinely empty BAL which would be stored as 0xc0. + // We have the block header but no source produced a BAL + // (pre-Amsterdam, pruned, or regenerator absent/declined). + // Return 0x80 — unambiguously "not available", distinct from + // a genuinely empty BAL which would be 0xc0. bals = append(bals, notAvailableSentinel) bytes += len(notAvailableSentinel) continue diff --git a/p2p/protocols/eth/handlers_test.go b/p2p/protocols/eth/handlers_test.go index 218704dd91f..d9d6e086710 100644 --- a/p2p/protocols/eth/handlers_test.go +++ b/p2p/protocols/eth/handlers_test.go @@ -571,6 +571,8 @@ func (balHeaderReader) Integrity(context.Context) error { panic("not expected") // ethereum/EIPs#11553) for any hash we don't have stored — including unknown // blocks and known blocks with no BAL recorded. func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) { + rawdb.ResetBALCacheForTest() + t.Cleanup(rawdb.ResetBALCacheForTest) db := memdb.NewTestDB(t, dbcfg.ChainDB) tx, err := db.BeginRw(context.Background()) if err != nil { @@ -594,7 +596,7 @@ func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) } query := GetBlockAccessListsPacket{hashKnownWithBAL, hashUnknown, hashKnownNoBAL} - result := AnswerGetBlockAccessListsQuery(tx, query, reader) + result := AnswerGetBlockAccessListsQuery(context.Background(), tx, query, reader) if len(result) != 3 { t.Fatalf("result len: have %d, want 3", len(result)) @@ -613,6 +615,8 @@ func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) // TestAnswerGetBlockAccessListsQuery_SoftSizeLimit verifies the handler // respects softResponseLimit by truncating the response (not padding). func TestAnswerGetBlockAccessListsQuery_SoftSizeLimit(t *testing.T) { + rawdb.ResetBALCacheForTest() + t.Cleanup(rawdb.ResetBALCacheForTest) db := memdb.NewTestDB(t, dbcfg.ChainDB) tx, err := db.BeginRw(context.Background()) if err != nil { @@ -646,7 +650,7 @@ func TestAnswerGetBlockAccessListsQuery_SoftSizeLimit(t *testing.T) { query = append(query, h) } - result := AnswerGetBlockAccessListsQuery(tx, query, reader) + result := AnswerGetBlockAccessListsQuery(context.Background(), tx, query, reader) if len(result) < 1 || len(result) >= len(query) { t.Fatalf("expected truncation: have %d entries, want 1..%d", len(result), len(query)-1) } diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client.go b/p2p/sentry/sentry_multi_client/sentry_multi_client.go index ee1773813e6..178b89faadc 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client.go @@ -679,7 +679,7 @@ func (cs *MultiClient) getBlockAccessLists71(ctx context.Context, inreq *sentryp return err } defer tx.Rollback() - response := eth.AnswerGetBlockAccessListsQuery(tx, query.GetBlockAccessListsPacket, cs.blockReader) + response := eth.AnswerGetBlockAccessListsQuery(ctx, tx, query.GetBlockAccessListsPacket, cs.blockReader) tx.Rollback() b, err := rlp.EncodeToBytes(ð.BlockAccessListsPacket66{ RequestId: query.RequestId, From bdfb7569d4c24258837b09fc1cec227895e1d8ec Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:46:16 +0000 Subject: [PATCH 104/120] db/rawdb, exec, p2p: drop BAL MDBX writes, cache-only lookup with regenerator fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BAL was being written to MDBX on the NewPayload critical path — a multi-hundred-KB Put per block that, on a churned DB, took tens of seconds. This commit removes the persistent storage entirely: - BlockAccessListBytes is now a 2-tier lookup: in-memory LRU cache → installed BALRegenerator (re-executes the block to derive the BAL). The chaindata-DB tier is gone. - execmodule.InsertBlocks caches the sidecar BAL (was MDBX Put on the blockOverlay) — same call-site, cache-only. - bal-downloader caches fetched BALs (was MDBX rwDB.Update). The backward-scan dedup check now consults the cache too. Callers updated to use the cache-aware lookup (cache → regenerator): - execution/stagedsync/exec3.go: BAL validation in parallel exec path - execution/stagedsync/bal_create.go: ProcessBAL validator cross-check - execution/execmodule/getters.go: GetPayloadBodiesByHash/Range RPC - rpc/jsonrpc/eth_block_access_list.go: eth_getBlockAccessList RPC The eth/71 server-side handler (AnswerGetBlockAccessListsQuery) stays cache → regenerator only — explicitly NOT a peer-fetch relay, to avoid amplification of remote BAL requests. Tests: - db/rawdb/balcache_test.go: dropped the now-dead DBHitPromotesToCache test; the rest stay identical except for the dropped tx argument. - p2p/protocols/eth/handlers_test.go: test fixtures use rawdb.CacheBlockAccessList instead of WriteBlockAccessListBytes. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/rawdb/balcache.go | 45 +++++++--------- db/rawdb/balcache_test.go | 54 ++++--------------- execution/execmodule/getters.go | 8 +-- execution/execmodule/inserters.go | 8 +-- execution/stagedsync/bal_create.go | 14 ++--- execution/stagedsync/exec3.go | 11 ++-- p2p/protocols/eth/handlers.go | 6 +-- p2p/protocols/eth/handlers_test.go | 8 +-- .../sentry_multi_client/bal_downloader.go | 41 +++++--------- rpc/jsonrpc/eth_block_access_list.go | 2 +- 10 files changed, 69 insertions(+), 128 deletions(-) diff --git a/db/rawdb/balcache.go b/db/rawdb/balcache.go index 3b1d3d79ac1..33e5da03027 100644 --- a/db/rawdb/balcache.go +++ b/db/rawdb/balcache.go @@ -23,7 +23,6 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/db/kv" ) // balCacheSize is how many recent blocks' EIP-7928 BlockAccessLists are kept in @@ -56,10 +55,11 @@ func init() { // the lookup path here installs one via SetBALRegenerator and consults it as a // fallback when neither the in-memory cache nor the chaindata DB has the BAL. // -// The returned bytes are the canonical RLP encoding (as written by -// rawdb.WriteBlockAccessListBytes) so callers can hand them to peers -// unchanged. The hash of the decoded BAL MUST match header.BlockAccessListHash -// — peers verify this on receipt; a mismatch fails the eth/71 BAL response. +// The returned bytes are the canonical RLP encoding (matching what +// types.EncodeBlockAccessListBytes produces) so callers can hand them to +// peers unchanged. The hash of the decoded BAL MUST match +// header.BlockAccessListHash — peers verify this on receipt; a mismatch +// fails the eth/71 BAL response. type BALRegenerator interface { RegenerateBlockAccessList(ctx context.Context, hash common.Hash, number uint64) ([]byte, error) } @@ -104,34 +104,29 @@ func CachedBlockAccessList(hash common.Hash) ([]byte, bool) { return balCache.Get(hash) } -// BlockAccessListBytes returns the BAL bytes for (hash, number). Lookup order: -// 1. In-memory LRU cache (recent blocks, freshly produced). -// 2. Chaindata DB (BALs the eth/71 bal-downloader fetched from peers, or -// legacy persisted BALs). -// 3. If a BALRegenerator is installed, re-execute the block to derive the BAL. -// The result is added to the cache before returning. +// BlockAccessListBytes returns the BAL bytes for (hash, number). Two-tier +// lookup: +// 1. In-memory LRU cache (recent blocks, freshly produced or recently +// regenerated). +// 2. If a BALRegenerator is installed, re-execute the block to derive the +// BAL. The result is added to the cache before returning. +// +// The BAL is intentionally NOT persisted to MDBX. Writing the BAL on the +// NewPayload critical path was a multi-hundred-KB Put per block that, on a +// churned DB, took tens of seconds. The cache + regenerate-on-miss model +// removes the write entirely; older blocks needed by peers (or RPC) are +// reconstructed on demand. // // Returns (nil, nil) if no source has it (no regenerator installed, or -// regenerator returned nil). Returns a non-nil error only on actual lookup -// failure (DB error, regenerator error). +// regenerator returned nil). Returns a non-nil error only on actual +// regenerator failure. // // Callers passing the context (ctx) get cancellation of the regeneration leg // — peer-driven serving paths should respect the peer's read deadline. -func BlockAccessListBytes(ctx context.Context, db kv.Getter, hash common.Hash, number uint64) ([]byte, error) { +func BlockAccessListBytes(ctx context.Context, hash common.Hash, number uint64) ([]byte, error) { if v, ok := balCache.Get(hash); ok { return v, nil } - stored, err := ReadBlockAccessListBytes(db, hash, number) - if err != nil { - return nil, err - } - if stored != nil { - // Promote DB hits into the cache so the next lookup is a single - // LRU probe — important for serving the same hash repeatedly - // (peers re-asking, RPC retries). - CacheBlockAccessList(hash, stored) - return stored, nil - } regen := getBALRegenerator() if regen == nil { return nil, nil diff --git a/db/rawdb/balcache_test.go b/db/rawdb/balcache_test.go index cf2dca18e36..061d41ec9e1 100644 --- a/db/rawdb/balcache_test.go +++ b/db/rawdb/balcache_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/db/kv/memdb" "github.com/erigontech/erigon/db/rawdb" ) @@ -33,7 +32,7 @@ import ( type fakeRegenerator struct { calls atomic.Int32 // resultFor maps the (hash) → bytes the regenerator will return. Unset - // hashes get the default canned bytes if set, else (nil, errNoBALForHash). + // hashes get the default canned bytes if set, else (nil, nil). resultFor map[common.Hash][]byte defaultBytes []byte errAlways error @@ -66,8 +65,6 @@ func hashFromByte(b byte) common.Hash { func TestBlockAccessListBytes_CacheHitShortCircuits(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x01) data := []byte{0xc1, 0x00} @@ -76,50 +73,27 @@ func TestBlockAccessListBytes_CacheHitShortCircuits(t *testing.T) { // Install a regenerator that, if called, fails the test. rawdb.SetBALRegenerator(&fakeRegenerator{errAlways: errFakeRegen}) - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 7) + got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 7) require.NoError(t, err) require.Equal(t, data, got) } -func TestBlockAccessListBytes_DBHitPromotesToCache(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() - - hash := hashFromByte(0x02) - data := []byte{0xc2, 0xff, 0xee} - require.NoError(t, rawdb.WriteBlockAccessListBytes(tx, hash, 11, data)) - - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 11) - require.NoError(t, err) - require.Equal(t, data, got) - - // Subsequent lookup must be a cache hit (no DB read necessary). - rawdb.SetBALRegenerator(nil) - got2, ok := rawdb.CachedBlockAccessList(hash) - require.True(t, ok, "DB-source BAL must be promoted to cache") - require.Equal(t, data, got2) -} - func TestBlockAccessListBytes_RegeneratorFallback(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x03) regenerated := []byte{0xc3, 0x42} regen := &fakeRegenerator{defaultBytes: regenerated} rawdb.SetBALRegenerator(regen) - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 22) + got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 22) require.NoError(t, err) require.Equal(t, regenerated, got) require.Equal(t, int32(1), regen.calls.Load(), "regenerator should be called once on miss") // Repeated lookup hits the cache, regenerator NOT called again. - got2, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 22) + got2, err := rawdb.BlockAccessListBytes(context.Background(), hash, 22) require.NoError(t, err) require.Equal(t, regenerated, got2) require.Equal(t, int32(1), regen.calls.Load(), "cached regenerated BAL must short-circuit subsequent lookups") @@ -128,27 +102,23 @@ func TestBlockAccessListBytes_RegeneratorFallback(t *testing.T) { func TestBlockAccessListBytes_NoRegeneratorOnMiss(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x04) rawdb.SetBALRegenerator(nil) - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 33) + got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 33) require.NoError(t, err) - require.Nil(t, got, "no cache, no DB, no regenerator → nil bytes (peer sees 'not available')") + require.Nil(t, got, "no cache, no regenerator → nil bytes (peer sees 'not available')") } func TestBlockAccessListBytes_RegeneratorReturnsNil(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x05) regen := &fakeRegenerator{} // defaultBytes nil rawdb.SetBALRegenerator(regen) - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 44) + got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 44) require.NoError(t, err) require.Nil(t, got) require.Equal(t, int32(1), regen.calls.Load()) @@ -162,14 +132,12 @@ func TestBlockAccessListBytes_RegeneratorReturnsNil(t *testing.T) { func TestBlockAccessListBytes_RegeneratorError(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x06) regen := &fakeRegenerator{errAlways: errFakeRegen} rawdb.SetBALRegenerator(regen) - _, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 55) + _, err := rawdb.BlockAccessListBytes(context.Background(), hash, 55) require.ErrorIs(t, err, errFakeRegen) _, ok := rawdb.CachedBlockAccessList(hash) require.False(t, ok, "regenerator error must not pollute the cache") @@ -201,8 +169,6 @@ func TestCacheBlockAccessList_CopiesBytes(t *testing.T) { func TestSetBALRegenerator_ReplaceAndClear(t *testing.T) { t.Cleanup(rawdb.ResetBALCacheForTest) rawdb.ResetBALCacheForTest() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() hash := hashFromByte(0x09) r1 := &fakeRegenerator{defaultBytes: []byte{0xa1}} @@ -210,7 +176,7 @@ func TestSetBALRegenerator_ReplaceAndClear(t *testing.T) { rawdb.SetBALRegenerator(r1) rawdb.SetBALRegenerator(r2) // replace - got, err := rawdb.BlockAccessListBytes(context.Background(), tx, hash, 99) + got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 99) require.NoError(t, err) require.Equal(t, []byte{0xa2}, got) require.Zero(t, r1.calls.Load(), "old regenerator must not be called after replacement") @@ -218,7 +184,7 @@ func TestSetBALRegenerator_ReplaceAndClear(t *testing.T) { rawdb.ResetBALCacheForTest() // clear cache so the next lookup hits the regenerator again rawdb.SetBALRegenerator(nil) // explicit clear - got, err = rawdb.BlockAccessListBytes(context.Background(), tx, hash, 99) + got, err = rawdb.BlockAccessListBytes(context.Background(), hash, 99) require.NoError(t, err) require.Nil(t, got, "cleared regenerator → miss returns nil") } diff --git a/execution/execmodule/getters.go b/execution/execmodule/getters.go index 67b69f10433..c9010d82e56 100644 --- a/execution/execmodule/getters.go +++ b/execution/execmodule/getters.go @@ -264,9 +264,9 @@ func (e *ExecModule) GetPayloadBodiesByHash(ctx context.Context, hashes []common if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByHash: MarshalTransactionsBinary error %w", err) } - balBytes, err := rawdb.ReadBlockAccessListBytes(tx, h, *number) + balBytes, err := rawdb.BlockAccessListBytes(ctx, h, *number) if err != nil { - return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByHash: ReadBlockAccessListBytes error %w", err) + return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByHash: BlockAccessListBytes error %w", err) } var bal []byte if len(balBytes) > 0 { @@ -310,9 +310,9 @@ func (e *ExecModule) GetPayloadBodiesByRange(ctx context.Context, start, count u if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByRange: MarshalTransactionsBinary error %w", err) } - balBytes, err := rawdb.ReadBlockAccessListBytes(tx, hash, blockNum) + balBytes, err := rawdb.BlockAccessListBytes(ctx, hash, blockNum) if err != nil { - return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByRange: ReadBlockAccessListBytes error %w", err) + return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByRange: BlockAccessListBytes error %w", err) } var bal []byte if len(balBytes) > 0 { diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index f4a90d71201..5efa9f457e1 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -140,9 +140,11 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) if header.BlockAccessListHash == nil { return 0, fmt.Errorf("ethereumExecutionModule.InsertBlocks: block access list provided without hash for block %d", height) } - if err := rawdb.WriteBlockAccessListBytes(blockOverlay, header.Hash(), height, block.BlockAccessList); err != nil { - return 0, fmt.Errorf("ethereumExecutionModule.InsertBlocks: writeBlockAccessList, block %d: %s", height, err) - } + // BAL bytes go to the in-memory cache instead of MDBX. The + // chaindata write was tens of seconds per block on churned DBs; + // see db/rawdb/balcache.go. Older blocks needed by eth/71 peers + // or RPC are regenerated on demand via the BALRegenerator. + rawdb.CacheBlockAccessList(header.Hash(), block.BlockAccessList) } e.logger.Trace("Inserted block", "hash", header.Hash(), "number", header.Number) } diff --git a/execution/stagedsync/bal_create.go b/execution/stagedsync/bal_create.go index 9642a4a5e77..62e506f30ce 100644 --- a/execution/stagedsync/bal_create.go +++ b/execution/stagedsync/bal_create.go @@ -73,13 +73,13 @@ func ProcessBAL(tx kv.TemporalRwTx, h *types.Header, vio *state.VersionedIO, isE return fmt.Errorf("block %d: EIP-7928 active but BlockAccessListHash is nil in header", blockNum) } headerBALHash := *h.BlockAccessListHash - dbBALBytes, err := rawdb.ReadBlockAccessListBytes(tx, blockHash, blockNum) - if err != nil { - return fmt.Errorf("block %d: read stored block access list: %w", blockNum, err) - } - // BAL data may not be stored for blocks downloaded via backward - // block downloader (p2p sync) since it does not carry BAL sidecars. - // Remove after eth/71 has been implemented. + // BALs are cache-only (see db/rawdb/balcache.go). The sidecar BAL from + // engine_newPayload is cached by execmodule.InsertBlocks; the eth/71 + // bal-downloader also caches what it fetches from peers. Lookup that + // cached BAL here for the validator cross-check. Cache misses are OK — + // not every code path has a sidecar (backward block downloader doesn't + // carry one). + dbBALBytes, _ := rawdb.CachedBlockAccessList(blockHash) if dbBALBytes != nil { dbBAL, err := types.DecodeBlockAccessListBytes(dbBALBytes) if err != nil { diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5d2e668794c..0d2b61bd6c0 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -608,13 +608,10 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m } var dbBAL types.BlockAccessList - // Read BAL through blockTx (overlay or execRoTx) — do NOT open - // a separate db.View() as it can deadlock with the stageloop's - // RW transaction when BlockOverlay is active. - data, err := rawdb.ReadBlockAccessListBytes(blockTx, b.Hash(), blockNum) - if err != nil { - return err - } + // BALs are cache-only (see db/rawdb/balcache.go). If the engine_newPayload + // path cached one for this block (via execmodule.InsertBlocks), pick it up + // here for the parallel exec's BAL validation. + data, _ := rawdb.CachedBlockAccessList(b.Hash()) if len(data) > 0 && !dbg.IgnoreBAL { dbBAL, err = types.DecodeBlockAccessListBytes(data) if err != nil { diff --git a/p2p/protocols/eth/handlers.go b/p2p/protocols/eth/handlers.go index 99dfacb07e9..efa6568d05e 100644 --- a/p2p/protocols/eth/handlers.go +++ b/p2p/protocols/eth/handlers.go @@ -207,9 +207,9 @@ func AnswerGetBlockAccessListsQuery(ctx context.Context, db kv.Tx, query GetBloc bytes += len(notAvailableSentinel) continue } - // Cache-aware lookup: in-memory cache → chaindata DB → installed - // BALRegenerator (re-executes the block when nothing is stored). - bal, _ := rawdb.BlockAccessListBytes(ctx, db, hash, *number) + // 2-tier lookup: in-memory cache → installed BALRegenerator + // (re-executes the block when nothing is cached). No MDBX read. + bal, _ := rawdb.BlockAccessListBytes(ctx, hash, *number) if len(bal) == 0 { // We have the block header but no source produced a BAL // (pre-Amsterdam, pruned, or regenerator absent/declined). diff --git a/p2p/protocols/eth/handlers_test.go b/p2p/protocols/eth/handlers_test.go index d9d6e086710..71bb9a99a52 100644 --- a/p2p/protocols/eth/handlers_test.go +++ b/p2p/protocols/eth/handlers_test.go @@ -591,9 +591,7 @@ func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) } bal := []byte{0xc3, 0x01, 0x02, 0x03} // short valid RLP payload (non-empty) - if err := rawdb.WriteBlockAccessListBytes(tx, hashKnownWithBAL, 100, bal); err != nil { - t.Fatalf("WriteBlockAccessListBytes: %v", err) - } + rawdb.CacheBlockAccessList(hashKnownWithBAL, bal) query := GetBlockAccessListsPacket{hashKnownWithBAL, hashUnknown, hashKnownNoBAL} result := AnswerGetBlockAccessListsQuery(context.Background(), tx, query, reader) @@ -644,9 +642,7 @@ func TestAnswerGetBlockAccessListsQuery_SoftSizeLimit(t *testing.T) { h := common.Hash{byte(i + 1)} num := uint64(1000 + i) reader[h] = num - if err := rawdb.WriteBlockAccessListBytes(tx, h, num, bal); err != nil { - t.Fatalf("WriteBlockAccessListBytes: %v", err) - } + rawdb.CacheBlockAccessList(h, bal) query = append(query, h) } diff --git a/p2p/sentry/sentry_multi_client/bal_downloader.go b/p2p/sentry/sentry_multi_client/bal_downloader.go index c711d71656c..c2451b91d57 100644 --- a/p2p/sentry/sentry_multi_client/bal_downloader.go +++ b/p2p/sentry/sentry_multi_client/bal_downloader.go @@ -198,17 +198,10 @@ func (d *BALDownloader) collectMissingBALs(ctx context.Context) ([]missingBAL, e break } hash := hdr.Hash() - existing, err := rawdb.ReadBlockAccessListBytes(tx, hash, n) - if err != nil { - // A real DB error here used to silently fall through, treating - // this slot as "missing" — leading to refetch+rewrite+refetch - // loops on every scan pass against a transient or persistent - // DB issue. Log and skip; next scan will retry. - d.logger.Debug("[bal-downloader] ReadBlockAccessListBytes failed, skipping slot", - "n", n, "hash", hash, "err", err) - continue - } - if len(existing) > 0 { + // Cache-only — BALs are not persisted to MDBX. If the cache has + // already absorbed this hash (recently produced locally or fetched + // from another peer this run), skip the fetch. + if _, ok := rawdb.CachedBlockAccessList(hash); ok { continue } missing = append(missing, missingBAL{ @@ -286,27 +279,19 @@ func (d *BALDownloader) fetchBatch(ctx context.Context, peer [64]byte, sentryI i return } - // Write accepted entries. The fetcher already decoded EIP-8159 (post + // Cache accepted entries. The fetcher already decoded EIP-8159 (post // ethereum/EIPs#11553) sentinels: nil = "peer doesn't have it" (was 0x80 // on the wire) — skip and retry next pass from another peer; {0xc0} = - // "genuinely empty BAL, hash-verified" — write the canonical RLP so - // callers that distinguish "have" vs "don't have" via rawdb see the - // record; anything else = hash-validated BAL bytes. + // "genuinely empty BAL, hash-verified"; anything else = hash-validated + // BAL bytes. We cache rather than write to MDBX — see + // db/rawdb/balcache.go for the rationale. var stored int - if err := d.rwDB.Update(ctx, func(tx kv.RwTx) error { - for i, payload := range got { - if len(payload) == 0 { - continue - } - if err := rawdb.WriteBlockAccessListBytes(tx, batch[i].hash, batch[i].number, payload); err != nil { - return err - } - stored++ + for i, payload := range got { + if len(payload) == 0 { + continue } - return nil - }); err != nil { - d.logger.Debug("[bal-downloader] db write failed", "err", err, "batch_size", len(batch)) - return + rawdb.CacheBlockAccessList(batch[i].hash, payload) + stored++ } if stored > 0 { diff --git a/rpc/jsonrpc/eth_block_access_list.go b/rpc/jsonrpc/eth_block_access_list.go index 041fb6752a4..e516cba161a 100644 --- a/rpc/jsonrpc/eth_block_access_list.go +++ b/rpc/jsonrpc/eth_block_access_list.go @@ -66,7 +66,7 @@ func (api *APIImpl) GetBlockAccessList(ctx context.Context, numberOrHash rpc.Blo } } - data, err := rawdb.ReadBlockAccessListBytes(tx, blockHash, blockNum) + data, err := rawdb.BlockAccessListBytes(ctx, blockHash, blockNum) if err != nil { return nil, err } From c9664ae23bc390f2cc53448fa39c5199f9fa9712 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:50:57 +0000 Subject: [PATCH 105/120] execmodule, node/eth: refactor BAL regenerator on ComputeBlockContext; wire at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit balregen.go now uses transactions.ComputeBlockContext to set up the IBS rooted at the parent state — same machinery the RPC tracing / receipts generation paths use, so the BAL replay sees the same read-tracking layout the BAL hash assumes. The previous NewParentStateReader factory abstraction is gone; deps shrink to (DB, ChainConfig, Engine, BlockReader, TxNumsReader, Logger). node/eth/backend.go installs the regenerator via rawdb.SetBALRegenerator right after the exec module is constructed. From this point on, BlockAccessListBytes lookup of a cache-miss block will re-execute it to produce the BAL — used by eth/71 GetBlockAccessLists serving, eth_getBlockAccessList RPC, and the engine GetPayloadBodiesByHash/Range RPCs. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/execmodule/balregen.go | 164 ++++++++++++++----------------- node/eth/backend.go | 13 +++ 2 files changed, 88 insertions(+), 89 deletions(-) diff --git a/execution/execmodule/balregen.go b/execution/execmodule/balregen.go index 535615345be..b816f2bf0f1 100644 --- a/execution/execmodule/balregen.go +++ b/execution/execmodule/balregen.go @@ -24,42 +24,43 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" + rawdbv3 "github.com/erigontech/erigon/db/kv/rawdbv3" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/vm/evmtypes" "github.com/erigontech/erigon/execution/protocol" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/execution/types/accounts" "github.com/erigontech/erigon/execution/vm" + "github.com/erigontech/erigon/rpc/transactions" ) // BALRegeneratorDeps bundles the per-node helpers the regenerator needs to // re-execute a historical block: chain config, consensus engine, block reader -// (for txs/bodies), and a state-reader factory keyed on block hash+number. -// -// The state-reader factory returns a reader rooted at the *parent* state of the -// requested block. Caller owns the returned closer (typically a kv.Tx.Rollback) -// — the regenerator invokes it once after the re-execution finishes. +// (for txs/bodies/headers), txNum reader (for state-at-block resolution), and +// the temporal RO DB. type BALRegeneratorDeps struct { - ChainConfig *chain.Config - Engine rules.Engine - BlockReader services.FullBlockReader - NewParentStateReader func(ctx context.Context, blockHash common.Hash, blockNum uint64) (state.StateReader, services.HeaderReader, func(n uint64) (common.Hash, error), func(), error) - Logger log.Logger + DB kv.TemporalRoDB + ChainConfig *chain.Config + Engine rules.Engine + BlockReader services.FullBlockReader + TxNumsReader rawdbv3.TxNumsReader + Logger log.Logger } // BALRegenerator implements rawdb.BALRegenerator by re-executing the requested -// block against its parent state with VersionMap-enabled IBS read tracking. The -// computed BAL is RLP-encoded and returned; the hash of the decoded BAL must -// match header.BlockAccessListHash (the parallel-exec path that produced the -// original BAL uses the same VersionedIO tracking, so the bytes are -// expected to be hash-equivalent for a non-failing execution). +// block against its parent state with VersionMap-enabled IBS read tracking. +// Uses transactions.ComputeBlockContext to construct a state reader rooted at +// the parent state — same approach as RPC tracing / receipts generation, so +// we get the read-tracking layout the BAL hash assumes without duplicating +// the state-at-block resolution code. // // Suitable for serving eth/71 GetBlockAccessLists when the local node hasn't -// stored the BAL (pre-Amsterdam, pruned, never-received, or the cache window -// has rolled). Does NOT modify any persistent state — the IBS writes go to -// state.NewNoopWriter(). +// cached the BAL (the block was produced before this node started, or the +// cache window rolled past it). Does NOT modify any persistent state — IBS +// writes go to state.NewNoopWriter(). type BALRegenerator struct { deps BALRegeneratorDeps } @@ -68,26 +69,18 @@ func NewBALRegenerator(deps BALRegeneratorDeps) *BALRegenerator { return &BALRegenerator{deps: deps} } -// RegenerateBlockAccessList re-executes the block at (hash, number) and returns -// the RLP-encoded BAL. Returns (nil, nil) when the block can't be located OR -// the chain config doesn't have BAL active at the block's timestamp. +// RegenerateBlockAccessList re-executes the block at (hash, number) and +// returns the RLP-encoded BAL. Returns (nil, nil) when the block can't be +// located, body is pruned, or the chain config doesn't have BAL active at +// the block's timestamp. func (r *BALRegenerator) RegenerateBlockAccessList(ctx context.Context, hash common.Hash, number uint64) ([]byte, error) { - if r.deps.NewParentStateReader == nil { - return nil, fmt.Errorf("BALRegenerator: NewParentStateReader is nil") - } - stateReader, headerReader, blockHashFunc, closer, err := r.deps.NewParentStateReader(ctx, hash, number) + tx, err := r.deps.DB.BeginTemporalRo(ctx) if err != nil { - return nil, fmt.Errorf("BALRegenerator: parent-state reader: %w", err) - } - if closer != nil { - defer closer() + return nil, fmt.Errorf("BALRegenerator: BeginTemporalRo: %w", err) } + defer tx.Rollback() - var stateGetter kv.TemporalGetter - _ = stateGetter // future: when the reader needs explicit getter passthrough - - // Fetch the block from the local reader (txs + header). - header, err := r.deps.BlockReader.Header(ctx, dbForReader(headerReader), hash, number) + header, err := r.deps.BlockReader.Header(ctx, tx, hash, number) if err != nil { return nil, fmt.Errorf("BALRegenerator: header lookup: %w", err) } @@ -98,67 +91,59 @@ func (r *BALRegenerator) RegenerateBlockAccessList(ctx context.Context, hash com // Pre-Amsterdam blocks don't have BALs by spec. return nil, nil } - body, err := r.deps.BlockReader.BodyWithTransactions(ctx, dbForReader(headerReader), hash, number) + body, err := r.deps.BlockReader.BodyWithTransactions(ctx, tx, hash, number) if err != nil { return nil, fmt.Errorf("BALRegenerator: body lookup: %w", err) } if body == nil { + // Body pruned — we can't re-execute. return nil, nil } block := types.NewBlockFromStorage(hash, header, body.Transactions, body.Uncles, body.Withdrawals) - bal, err := computeBlockAccessList(ctx, r.deps.ChainConfig, r.deps.Engine, block, stateReader, headerReader, blockHashFunc, r.deps.Logger) + // ComputeBlockContext at txIndex=0 returns an IBS reading from the + // state BEFORE the block's first transaction (= post-state of the + // parent block). Same machinery the RPC tracing path uses. + ibs, blockCtx, _, vmRules, signer, err := transactions.ComputeBlockContext(ctx, r.deps.Engine, header, r.deps.ChainConfig, r.deps.BlockReader, nil, r.deps.TxNumsReader, tx, 0) if err != nil { - return nil, fmt.Errorf("BALRegenerator: re-exec block %d: %w", number, err) + return nil, fmt.Errorf("BALRegenerator: ComputeBlockContext: %w", err) + } + // IBS read-tracking requires a VersionMap — versionedRead is the + // only path that populates ibs.versionedReads, which TxIO() reads + // to build the BAL. + ibs.SetVersionMap(state.NewVersionMap(nil)) + + bal, err := replayBlockForBAL(ctx, r.deps.ChainConfig, r.deps.Engine, block, &blockCtx, vmRules, signer, ibs, r.deps.Logger) + if err != nil { + return nil, fmt.Errorf("BALRegenerator: replay block %d: %w", number, err) } if bal == nil { return nil, nil } - balBytes, err := types.EncodeBlockAccessListBytes(bal) - if err != nil { - return nil, fmt.Errorf("BALRegenerator: encode: %w", err) - } - return balBytes, nil + return types.EncodeBlockAccessListBytes(bal) } -// dbForReader is a placeholder for the kv.Getter context the HeaderReader/ -// BodyWithTransactions paths sometimes need. The closure inside -// NewParentStateReader is expected to share its tx with the header reader; this -// is the seam where it's threaded through. -// -// TODO: the BlockReader's Header/BodyWithTransactions need an explicit kv.Tx — -// NewParentStateReader currently exposes only a state.StateReader. Surface the -// tx alongside the state reader so this becomes a real argument. -func dbForReader(_ services.HeaderReader) kv.Getter { - return nil -} - -// computeBlockAccessList runs the block transactions through a simple IBS with -// VersionMap-enabled read tracking and returns the accumulated BAL. Mirrors the -// per-tx Merge pattern used by the block assembler — no parallel exec needed. -func computeBlockAccessList( +// replayBlockForBAL drives the per-tx loop, merging the IBS's TxIO into a +// per-block VersionedIO. Mirrors the block-assembler pattern — no parallel +// exec, no state writer. Returns the accumulated BAL after the last tx. +func replayBlockForBAL( ctx context.Context, chainConfig *chain.Config, engine rules.Engine, block *types.Block, - stateReader state.StateReader, - headerReader services.HeaderReader, - blockHashFunc func(n uint64) (common.Hash, error), + blockCtx *evmtypes.BlockContext, + vmRules *chain.Rules, + signer *types.Signer, + ibs *state.IntraBlockState, logger log.Logger, ) (types.BlockAccessList, error) { - ibs := state.New(stateReader) - defer ibs.Release(false) - // Read tracking requires a VersionMap — versionedRead is the only path - // that populates ibs.versionedReads. An empty VersionMap is fine; we - // don't need cross-tx coordination here. - ibs.SetVersionMap(state.NewVersionMap(nil)) - header := block.HeaderNoCopy() gp := new(protocol.GasPool).AddGas(block.GasLimit()).AddBlobGas(chainConfig.GetMaxBlobGasPerBlock(block.Time())) gasUsed := new(protocol.GasUsed) - chainReader := newChainReaderShim(chainConfig, headerReader) - + // chainReaderShim only exposes Config() — InitializeBlockExecution's + // system-init paths don't reach for parent headers here. + chainReader := &chainReaderShim{cfg: chainConfig} if err := protocol.InitializeBlockExecution(engine, chainReader, header, chainConfig, ibs, state.NewNoopWriter(), logger, nil); err != nil { return nil, fmt.Errorf("InitializeBlockExecution: %w", err) } @@ -167,12 +152,22 @@ func computeBlockAccessList( balIO = *balIO.Merge(ibs.TxIO()) ibs.ResetVersionedIO() + _ = blockCtx + _ = vmRules + _ = signer + + blockHashFn := protocol.GetHashFn(header, func(common.Hash, uint64) (*types.Header, error) { + // BAL re-execution doesn't need cross-block BLOCKHASH lookups — + // the relevant headers are already in blockCtx for this block. + return nil, nil + }) + for i, txn := range block.Transactions() { if err := ctx.Err(); err != nil { return nil, err } ibs.SetTxContext(block.NumberU64(), i) - _, err := protocol.ApplyTransaction(chainConfig, blockHashFunc, engine, accounts.NilAddress, gp, ibs, state.NewNoopWriter(), header, txn, gasUsed, vm.Config{NoReceipts: true}) + _, err := protocol.ApplyTransaction(chainConfig, blockHashFn, engine, accounts.NilAddress, gp, ibs, state.NewNoopWriter(), header, txn, gasUsed, vm.Config{NoReceipts: true}) if err != nil { return nil, fmt.Errorf("apply tx %d (%x): %w", i, txn.Hash(), err) } @@ -180,30 +175,21 @@ func computeBlockAccessList( ibs.ResetVersionedIO() } - // Finalize step is intentionally omitted: the engine.Finalize signatures - // differ across forks and require receipts/systemCall hooks we don't - // reconstruct here. Withdrawal + system-call accesses are typically - // captured during InitializeBlockExecution; any post-tx finalize writes - // would only matter for accounts already in the BAL from the tx loop. - // If the resulting BAL hash disagrees with header.BlockAccessListHash - // on a downstream peer's verification, that's the signal to add finalize - // support here (with the proper system-call hooks). + // Finalize-stage system writes (withdrawals, BeaconRoot, etc.) are + // captured during InitializeBlockExecution; engine.Finalize signatures + // vary by fork and would require receipts + system-call hooks we don't + // reconstruct here. If a downstream hash check flags a mismatch on + // finalize-touched accounts, that's the signal to add fork-aware + // finalize support to this path. return balIO.AsBlockAccessList(), nil } // chainReaderShim is a minimal adapter for rules.ChainHeaderReader that -// protocol.InitializeBlockExecution requires. The header-lookup methods stay -// nil because system-init calls in this path don't reach for parent headers -// (the only thing that would need them is engine.Finalize, which we don't -// invoke here). +// protocol.InitializeBlockExecution requires. The header-lookup methods +// stay nil because the BAL replay path doesn't need parent headers. type chainReaderShim struct { - cfg *chain.Config - reader services.HeaderReader -} - -func newChainReaderShim(cfg *chain.Config, reader services.HeaderReader) *chainReaderShim { - return &chainReaderShim{cfg: cfg, reader: reader} + cfg *chain.Config } func (s *chainReaderShim) Config() *chain.Config { return s.cfg } diff --git a/node/eth/backend.go b/node/eth/backend.go index 53bac782138..97bdd1c4ce9 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -1031,6 +1031,19 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger ) backend.execModule.SetPublishedSD(backend.notifications.Events.LatestSD) + // Install the BAL regenerator so eth/71 GetBlockAccessLists serving can + // fall back to re-executing the block when nothing is cached. BAL bytes + // are no longer persisted to MDBX (see db/rawdb/balcache.go), so older + // blocks needed by peers or RPC must be reconstructed on demand. + rawdb.SetBALRegenerator(execmodule.NewBALRegenerator(execmodule.BALRegeneratorDeps{ + DB: backend.chainDB, + ChainConfig: chainConfig, + Engine: backend.engine, + BlockReader: blockReader, + TxNumsReader: blockReader.TxnumReader(), + Logger: logger, + })) + var executionEngine executionclient.ExecutionEngine executionEngine, err = executionclient.NewExecutionClientDirect(chainreader.NewChainReaderEth1(chainConfig, backend.execModule, config.FcuTimeout), txPoolRpcClient) From f4901b5fb034b7221c03047ed40a72b4c44fbe14 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 16:57:34 +0000 Subject: [PATCH 106/120] rawdb, p2p/eth, engine_block_downloader: opportunistic BAL fetch during block sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the engine_block_downloader pulls blocks backwards from peers (catch-up sync, segment recovery), each batch is now followed by a non-blocking eth/71 GetBlockAccessLists request for those blocks. Hash-verified BALs populate the in-memory cache so the subsequent exec stage avoids local re-execution to derive them. - db/rawdb/balcache.go: new BALSyncFetcher interface + SetBALSyncFetcher / GetBALSyncFetcher. Mirrors the BALRegenerator plumbing (atomic.Pointer slot, nil-safe). - p2p/sentry/sentry_multi_client/bal_downloader.go: BALDownloader implements BALSyncFetcher.FetchBALs — picks an eth/71 peer, batches the request, caches every hash-validated response. Non-blocking by intent; failures fall through to the BALRegenerator on later lookup. - node/eth/backend.go: SetBALSyncFetcher(balDownloader) at startup alongside the existing background scan. - execution/engineapi/engine_block_downloader/block_downloader.go: after each feed.Next batch, kick a goroutine to FetchBALs for the batch's Amsterdam blocks (BlockAccessListHash != nil). engine_newPayload is intentionally NOT updated: post-Amsterdam the Engine API spec requires the CL to include the BAL bytes in the payload; a missing BAL is rejected as InvalidParamsError. Catch-up sync (where blocks arrive without BAL bodies) is covered by the engine_block_downloader hook above. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/rawdb/balcache.go | 48 ++++++++++++++++++- .../block_downloader.go | 29 +++++++++++ node/eth/backend.go | 11 +++-- .../sentry_multi_client/bal_downloader.go | 42 ++++++++++++++++ .../sentry_multi_client_test.go | 6 +-- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/db/rawdb/balcache.go b/db/rawdb/balcache.go index 33e5da03027..2ed4244095a 100644 --- a/db/rawdb/balcache.go +++ b/db/rawdb/balcache.go @@ -41,6 +41,13 @@ type balRegeneratorSlot struct{ r BALRegenerator } var balRegenerator atomic.Pointer[balRegeneratorSlot] +// balSyncFetcher holds the optionally-installed eth/71 peer-fetch backend used +// by sync paths to opportunistically fill the cache with peer-fetched BALs +// before falling back to local re-execution. +type balSyncFetcherSlot struct{ f BALSyncFetcher } + +var balSyncFetcher atomic.Pointer[balSyncFetcherSlot] + func init() { var err error balCache, err = lru.New[common.Hash, []byte](balCacheSize) @@ -84,6 +91,43 @@ func getBALRegenerator() BALRegenerator { return slot.r } +// BALSyncFetcher is invoked by sync paths (engine_block_downloader, +// engine_newPayload) to opportunistically fetch BALs from eth/71 peers and +// cache them. Implementations are non-blocking: they queue work and return +// quickly; sync stages don't wait on the fetch. +// +// hashes / numbers / expected are aligned positionally — entry i refers to +// the block at hashes[i] with number numbers[i], whose header committed to +// BAL hash expected[i]. Each successful fetch is hash-verified against +// expected before being cached (matches the spec for eth/71 GetBlockAccessLists +// responses). +// +// A missing or nil BALSyncFetcher is fine — the cache will fall through to +// the BALRegenerator on a later lookup. +type BALSyncFetcher interface { + FetchBALs(ctx context.Context, hashes []common.Hash, numbers []uint64, expected []common.Hash) +} + +// SetBALSyncFetcher installs the eth/71 peer-fetch backend used by sync +// hooks. Pass nil to clear. Safe to call multiple times. +func SetBALSyncFetcher(f BALSyncFetcher) { + if f == nil { + balSyncFetcher.Store(nil) + return + } + balSyncFetcher.Store(&balSyncFetcherSlot{f: f}) +} + +// GetBALSyncFetcher returns the installed fetcher, or nil if none is set. +// Sync hooks call this and dispatch when non-nil. +func GetBALSyncFetcher() BALSyncFetcher { + slot := balSyncFetcher.Load() + if slot == nil { + return nil + } + return slot.f +} + // CacheBlockAccessList stores the block's RLP-encoded BAL bytes in the in-memory // cache instead of persisting them to the chaindata DB. Writing the BAL to MDBX // on the NewPayload critical path was a multi-hundred-KB Put per block that, on a @@ -142,8 +186,10 @@ func BlockAccessListBytes(ctx context.Context, hash common.Hash, number uint64) return generated, nil } -// ResetBALCacheForTest clears the in-memory cache and the regenerator. Test-only. +// ResetBALCacheForTest clears the in-memory cache, the regenerator, and the +// sync fetcher. Test-only. func ResetBALCacheForTest() { balCache.Purge() balRegenerator.Store(nil) + balSyncFetcher.Store(nil) } diff --git a/execution/engineapi/engine_block_downloader/block_downloader.go b/execution/engineapi/engine_block_downloader/block_downloader.go index 3517c86bb49..3ccaa120238 100644 --- a/execution/engineapi/engine_block_downloader/block_downloader.go +++ b/execution/engineapi/engine_block_downloader/block_downloader.go @@ -29,6 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/execmodule" @@ -215,6 +216,17 @@ func (e *EngineBlockDownloader) downloadBlocks(ctx context.Context, req Backward default: e.logger.Trace("[EngineBlockDownloader] processing downloaded blocks", progressLogArgs...) } + // Opportunistically ask eth/71 peers for the BALs of the blocks we + // just downloaded so they're cached by the time the exec stage + // runs (avoids local BAL regeneration). Non-blocking: if no + // eth/71 peer is available or the request fails, the cache miss + // will fall through to the BALRegenerator later. + if fetcher := rawdb.GetBALSyncFetcher(); fetcher != nil { + hashes, numbers, expected := collectBALFetchRequests(blocks) + if len(hashes) > 0 { + go fetcher.FetchBALs(ctx, hashes, numbers, expected) + } + } err := e.chainRW.InsertBlocksAndWait(ctx, blocks) if err != nil { return err @@ -289,3 +301,20 @@ func (e *EngineBlockDownloader) execDownloadedBatch(ctx context.Context, block * } return nil } + +// collectBALFetchRequests pulls (hash, number, expectedBALHash) for every +// block in the batch whose header commits to a BAL. Pre-Amsterdam blocks +// (BlockAccessListHash == nil) are skipped. Returns positionally-aligned +// slices for rawdb.BALSyncFetcher.FetchBALs. +func collectBALFetchRequests(blocks []*types.Block) (hashes []common.Hash, numbers []uint64, expected []common.Hash) { + for _, b := range blocks { + h := b.HeaderNoCopy() + if h == nil || h.BlockAccessListHash == nil { + continue + } + hashes = append(hashes, b.Hash()) + numbers = append(numbers, b.NumberU64()) + expected = append(expected, *h.BlockAccessListHash) + } + return +} diff --git a/node/eth/backend.go b/node/eth/backend.go index 97bdd1c4ce9..18a753c8047 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -728,9 +728,14 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger if chainConfig.AmsterdamTime != nil { // Always-on once gated, negotiation-driven: if no peer advertises eth/71 // this is a silent no-op per scan pass. When eth/71 peers connect, the - // downloader backfills missing BALs into rawdb so subsequent stage_exec - // runs can skip local BAL regeneration. - go sentry_multi_client.NewBALDownloader(backend.sentryProvider.Client, backend.chainDB, logger).Run(backend.sentryCtx) + // downloader backfills missing BALs into the in-memory cache so + // subsequent serving (eth/71, RPC) can avoid local BAL regeneration. + balDownloader := sentry_multi_client.NewBALDownloader(backend.sentryProvider.Client, backend.chainDB, logger) + // Expose the on-demand FetchBALs entry point to sync hooks + // (engine_block_downloader after a batch, engine_newPayload when + // the CL payload doesn't carry a BAL). + rawdb.SetBALSyncFetcher(balDownloader) + go balDownloader.Run(backend.sentryCtx) } var txnProvider txnprovider.TxnProvider diff --git a/p2p/sentry/sentry_multi_client/bal_downloader.go b/p2p/sentry/sentry_multi_client/bal_downloader.go index c2451b91d57..1104fc38348 100644 --- a/p2p/sentry/sentry_multi_client/bal_downloader.go +++ b/p2p/sentry/sentry_multi_client/bal_downloader.go @@ -304,6 +304,48 @@ func (d *BALDownloader) fetchBatch(ctx context.Context, peer [64]byte, sentryI i } } +// FetchBALs implements rawdb.BALSyncFetcher: picks an eth/71 peer and +// requests BALs for the given (hash, number, expected) tuples in a single +// batched call, caching every hash-verified response. Non-blocking by intent +// — failures (no peer, peer declined, timeout) are silently dropped because +// the cache miss can still be filled later via the BALRegenerator. Sync +// stages MUST NOT block on this fetch. +// +// Splits the request into batches that stay below the eth/71 softResponseLimit +// (~2 MiB per response, ~24 BALs / batch). +func (d *BALDownloader) FetchBALs(ctx context.Context, hashes []common.Hash, numbers []uint64, expected []common.Hash) { + if len(hashes) == 0 { + return + } + if len(numbers) != len(hashes) || len(expected) != len(hashes) { + d.logger.Debug("[bal-downloader] FetchBALs called with misaligned slices", + "hashes", len(hashes), "numbers", len(numbers), "expected", len(expected)) + return + } + peer, sentryI, found, err := d.pickEth71Peer(ctx) + if err != nil || !found { + // No eth/71 peer available — caller can rely on the BALRegenerator + // when the cache miss surfaces. + return + } + const batchSize = 24 + for i := 0; i < len(hashes); i += batchSize { + end := i + batchSize + if end > len(hashes) { + end = len(hashes) + } + batch := make([]missingBAL, 0, end-i) + for j := i; j < end; j++ { + batch = append(batch, missingBAL{ + hash: hashes[j], + number: numbers[j], + expected: expected[j], + }) + } + d.fetchBatch(ctx, peer, sentryI, batch) + } +} + // pickEth71Peer iterates all sentries and returns a random peer that // advertises the eth/71 capability, plus the index of the sentry that peer // is connected via, or (_, _, false, nil) if none is connected. The sentry diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go index f388f46894d..b477ece2c7d 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go @@ -291,9 +291,9 @@ func TestGetBlockAccessLists71_AnswersAndSends(t *testing.T) { hashUnknown := common.Hash{0x02} const knownBlockNum uint64 = 100 bal := []byte{0xc3, 0x01, 0x02, 0x03} // short valid RLP non-empty payload - if err := rawdb.WriteBlockAccessListBytes(rwTx, hashKnown, knownBlockNum, bal); err != nil { - t.Fatalf("WriteBlockAccessListBytes: %v", err) - } + rawdb.ResetBALCacheForTest() + t.Cleanup(rawdb.ResetBALCacheForTest) + rawdb.CacheBlockAccessList(hashKnown, bal) if err := rwTx.Commit(); err != nil { t.Fatalf("commit: %v", err) } From 3ebe0333b623cedb6d50167f9d6e14c2bd132475 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 13 May 2026 17:05:35 +0000 Subject: [PATCH 107/120] =?UTF-8?q?balcache:=20move=20db/rawdb=20=E2=86=92?= =?UTF-8?q?=20execution/balcache=20(no=20DB,=20no=20rawdb=20home)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache is purely in-memory — no MDBX writes, no MDBX reads. Living under db/rawdb was a misnomer left over from the original implementation. New location is execution/balcache, neighbouring the types.BlockAccessList producers. No API renames; consumers swap rawdb.* → balcache.* prefixes. Drops the now-unused db/rawdb import from every consumer that only reached the package for the cache symbols. Co-Authored-By: Claude Opus 4.7 (1M context) --- {db/rawdb => execution/balcache}/balcache.go | 2 +- .../balcache}/balcache_test.go | 86 +++++++++---------- .../block_downloader.go | 6 +- execution/engineapi/engine_server_test.go | 12 +-- execution/execmodule/balregen.go | 2 +- execution/execmodule/getters.go | 5 +- execution/execmodule/inserters.go | 3 +- execution/stagedsync/bal_create.go | 4 +- execution/stagedsync/exec3.go | 3 +- node/eth/backend.go | 5 +- p2p/protocols/eth/handlers.go | 3 +- p2p/protocols/eth/handlers_test.go | 14 +-- .../sentry_multi_client/bal_downloader.go | 8 +- .../sentry_multi_client_test.go | 8 +- rpc/jsonrpc/eth_block_access_list.go | 4 +- 15 files changed, 86 insertions(+), 79 deletions(-) rename {db/rawdb => execution/balcache}/balcache.go (99%) rename {db/rawdb => execution/balcache}/balcache_test.go (67%) diff --git a/db/rawdb/balcache.go b/execution/balcache/balcache.go similarity index 99% rename from db/rawdb/balcache.go rename to execution/balcache/balcache.go index 2ed4244095a..afd371c4068 100644 --- a/db/rawdb/balcache.go +++ b/execution/balcache/balcache.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with Erigon. If not, see . -package rawdb +package balcache import ( "context" diff --git a/db/rawdb/balcache_test.go b/execution/balcache/balcache_test.go similarity index 67% rename from db/rawdb/balcache_test.go rename to execution/balcache/balcache_test.go index 061d41ec9e1..a4d7bc5d66d 100644 --- a/db/rawdb/balcache_test.go +++ b/execution/balcache/balcache_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with Erigon. If not, see . -package rawdb_test +package balcache_test import ( "context" @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" ) // fakeRegenerator records every regeneration request + serves canned responses. @@ -63,128 +63,128 @@ func hashFromByte(b byte) common.Hash { } func TestBlockAccessListBytes_CacheHitShortCircuits(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x01) data := []byte{0xc1, 0x00} - rawdb.CacheBlockAccessList(hash, data) + balcache.CacheBlockAccessList(hash, data) // Install a regenerator that, if called, fails the test. - rawdb.SetBALRegenerator(&fakeRegenerator{errAlways: errFakeRegen}) + balcache.SetBALRegenerator(&fakeRegenerator{errAlways: errFakeRegen}) - got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 7) + got, err := balcache.BlockAccessListBytes(context.Background(), hash, 7) require.NoError(t, err) require.Equal(t, data, got) } func TestBlockAccessListBytes_RegeneratorFallback(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x03) regenerated := []byte{0xc3, 0x42} regen := &fakeRegenerator{defaultBytes: regenerated} - rawdb.SetBALRegenerator(regen) + balcache.SetBALRegenerator(regen) - got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 22) + got, err := balcache.BlockAccessListBytes(context.Background(), hash, 22) require.NoError(t, err) require.Equal(t, regenerated, got) require.Equal(t, int32(1), regen.calls.Load(), "regenerator should be called once on miss") // Repeated lookup hits the cache, regenerator NOT called again. - got2, err := rawdb.BlockAccessListBytes(context.Background(), hash, 22) + got2, err := balcache.BlockAccessListBytes(context.Background(), hash, 22) require.NoError(t, err) require.Equal(t, regenerated, got2) require.Equal(t, int32(1), regen.calls.Load(), "cached regenerated BAL must short-circuit subsequent lookups") } func TestBlockAccessListBytes_NoRegeneratorOnMiss(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x04) - rawdb.SetBALRegenerator(nil) - got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 33) + balcache.SetBALRegenerator(nil) + got, err := balcache.BlockAccessListBytes(context.Background(), hash, 33) require.NoError(t, err) require.Nil(t, got, "no cache, no regenerator → nil bytes (peer sees 'not available')") } func TestBlockAccessListBytes_RegeneratorReturnsNil(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x05) regen := &fakeRegenerator{} // defaultBytes nil - rawdb.SetBALRegenerator(regen) + balcache.SetBALRegenerator(regen) - got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 44) + got, err := balcache.BlockAccessListBytes(context.Background(), hash, 44) require.NoError(t, err) require.Nil(t, got) require.Equal(t, int32(1), regen.calls.Load()) // A nil-from-regenerator must NOT be cached (so a later install of a // real regenerator can succeed). - _, ok := rawdb.CachedBlockAccessList(hash) + _, ok := balcache.CachedBlockAccessList(hash) require.False(t, ok, "nil regeneration result must not be cached") } func TestBlockAccessListBytes_RegeneratorError(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x06) regen := &fakeRegenerator{errAlways: errFakeRegen} - rawdb.SetBALRegenerator(regen) + balcache.SetBALRegenerator(regen) - _, err := rawdb.BlockAccessListBytes(context.Background(), hash, 55) + _, err := balcache.BlockAccessListBytes(context.Background(), hash, 55) require.ErrorIs(t, err, errFakeRegen) - _, ok := rawdb.CachedBlockAccessList(hash) + _, ok := balcache.CachedBlockAccessList(hash) require.False(t, ok, "regenerator error must not pollute the cache") } func TestCacheBlockAccessList_EmptyIsNoOp(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x07) - rawdb.CacheBlockAccessList(hash, nil) - rawdb.CacheBlockAccessList(hash, []byte{}) - _, ok := rawdb.CachedBlockAccessList(hash) + balcache.CacheBlockAccessList(hash, nil) + balcache.CacheBlockAccessList(hash, []byte{}) + _, ok := balcache.CachedBlockAccessList(hash) require.False(t, ok, "empty data must not be cached (would conflate with 'not available')") } func TestCacheBlockAccessList_CopiesBytes(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x08) src := []byte{0xde, 0xad, 0xbe, 0xef} - rawdb.CacheBlockAccessList(hash, src) + balcache.CacheBlockAccessList(hash, src) src[0] = 0xff // mutate caller's slice — cache must hold its own copy - got, ok := rawdb.CachedBlockAccessList(hash) + got, ok := balcache.CachedBlockAccessList(hash) require.True(t, ok) require.Equal(t, byte(0xde), got[0], "cache must defensively copy the input bytes") } func TestSetBALRegenerator_ReplaceAndClear(t *testing.T) { - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() hash := hashFromByte(0x09) r1 := &fakeRegenerator{defaultBytes: []byte{0xa1}} r2 := &fakeRegenerator{defaultBytes: []byte{0xa2}} - rawdb.SetBALRegenerator(r1) - rawdb.SetBALRegenerator(r2) // replace + balcache.SetBALRegenerator(r1) + balcache.SetBALRegenerator(r2) // replace - got, err := rawdb.BlockAccessListBytes(context.Background(), hash, 99) + got, err := balcache.BlockAccessListBytes(context.Background(), hash, 99) require.NoError(t, err) require.Equal(t, []byte{0xa2}, got) require.Zero(t, r1.calls.Load(), "old regenerator must not be called after replacement") require.Equal(t, int32(1), r2.calls.Load()) - rawdb.ResetBALCacheForTest() // clear cache so the next lookup hits the regenerator again - rawdb.SetBALRegenerator(nil) // explicit clear - got, err = rawdb.BlockAccessListBytes(context.Background(), hash, 99) + balcache.ResetBALCacheForTest() // clear cache so the next lookup hits the regenerator again + balcache.SetBALRegenerator(nil) // explicit clear + got, err = balcache.BlockAccessListBytes(context.Background(), hash, 99) require.NoError(t, err) require.Nil(t, got, "cleared regenerator → miss returns nil") } diff --git a/execution/engineapi/engine_block_downloader/block_downloader.go b/execution/engineapi/engine_block_downloader/block_downloader.go index 3ccaa120238..051f1ea62b1 100644 --- a/execution/engineapi/engine_block_downloader/block_downloader.go +++ b/execution/engineapi/engine_block_downloader/block_downloader.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/execmodule" @@ -221,7 +221,7 @@ func (e *EngineBlockDownloader) downloadBlocks(ctx context.Context, req Backward // runs (avoids local BAL regeneration). Non-blocking: if no // eth/71 peer is available or the request fails, the cache miss // will fall through to the BALRegenerator later. - if fetcher := rawdb.GetBALSyncFetcher(); fetcher != nil { + if fetcher := balcache.GetBALSyncFetcher(); fetcher != nil { hashes, numbers, expected := collectBALFetchRequests(blocks) if len(hashes) > 0 { go fetcher.FetchBALs(ctx, hashes, numbers, expected) @@ -305,7 +305,7 @@ func (e *EngineBlockDownloader) execDownloadedBatch(ctx context.Context, block * // collectBALFetchRequests pulls (hash, number, expectedBALHash) for every // block in the batch whose header commits to a BAL. Pre-Amsterdam blocks // (BlockAccessListHash == nil) are skipped. Returns positionally-aligned -// slices for rawdb.BALSyncFetcher.FetchBALs. +// slices for balcache.BALSyncFetcher.FetchBALs. func collectBALFetchRequests(blocks []*types.Block) (hashes []common.Hash, numbers []uint64, expected []common.Hash) { for _, b := range blocks { h := b.HeaderNoCopy() diff --git a/execution/engineapi/engine_server_test.go b/execution/engineapi/engine_server_test.go index 1fc483cf245..10c2db641cf 100644 --- a/execution/engineapi/engine_server_test.go +++ b/execution/engineapi/engine_server_test.go @@ -37,6 +37,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/kvcache" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/execmodule/execmoduletester" "github.com/erigontech/erigon/execution/tests/blockgen" @@ -302,12 +303,13 @@ func canonicalHashAt(t *testing.T, db kv.TemporalRoDB, blockNum uint64) common.H return hash } -func writeBlockAccessListBytes(t *testing.T, db kv.TemporalRwDB, blockHash common.Hash, blockNum uint64, balBytes []byte) { +func writeBlockAccessListBytes(t *testing.T, _ kv.TemporalRwDB, blockHash common.Hash, _ uint64, balBytes []byte) { t.Helper() - err := db.Update(context.Background(), func(tx kv.RwTx) error { - return rawdb.WriteBlockAccessListBytes(tx, blockHash, blockNum, balBytes) - }) - require.NoError(t, err) + // BALs are cache-only — the lookup path consults + // balcache.BlockAccessListBytes which checks the cache first, then the + // regenerator. Writing directly to the cache makes the test see exactly + // these bytes without re-executing. + balcache.CacheBlockAccessList(blockHash, balBytes) } func TestGetPayloadBodiesByHashV2(t *testing.T) { diff --git a/execution/execmodule/balregen.go b/execution/execmodule/balregen.go index b816f2bf0f1..bd690ca3b7c 100644 --- a/execution/execmodule/balregen.go +++ b/execution/execmodule/balregen.go @@ -50,7 +50,7 @@ type BALRegeneratorDeps struct { Logger log.Logger } -// BALRegenerator implements rawdb.BALRegenerator by re-executing the requested +// BALRegenerator implements balcache.BALRegenerator by re-executing the requested // block against its parent state with VersionMap-enabled IBS read tracking. // Uses transactions.ComputeBlockContext to construct a state reader rooted at // the parent state — same approach as RPC tracing / receipts generation, so diff --git a/execution/execmodule/getters.go b/execution/execmodule/getters.go index c9010d82e56..4225baabce0 100644 --- a/execution/execmodule/getters.go +++ b/execution/execmodule/getters.go @@ -27,6 +27,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/types" ) @@ -264,7 +265,7 @@ func (e *ExecModule) GetPayloadBodiesByHash(ctx context.Context, hashes []common if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByHash: MarshalTransactionsBinary error %w", err) } - balBytes, err := rawdb.BlockAccessListBytes(ctx, h, *number) + balBytes, err := balcache.BlockAccessListBytes(ctx, h, *number) if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByHash: BlockAccessListBytes error %w", err) } @@ -310,7 +311,7 @@ func (e *ExecModule) GetPayloadBodiesByRange(ctx context.Context, start, count u if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByRange: MarshalTransactionsBinary error %w", err) } - balBytes, err := rawdb.BlockAccessListBytes(ctx, hash, blockNum) + balBytes, err := balcache.BlockAccessListBytes(ctx, hash, blockNum) if err != nil { return nil, fmt.Errorf("ethereumExecutionModule.GetPayloadBodiesByRange: BlockAccessListBytes error %w", err) } diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index 5efa9f457e1..4507bb68bb4 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -24,6 +24,7 @@ import ( "github.com/holiman/uint256" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/commitment/commitmentdb" "github.com/erigontech/erigon/execution/metrics" @@ -144,7 +145,7 @@ func (e *ExecModule) InsertBlocks(ctx context.Context, blocks []*types.RawBlock) // chaindata write was tens of seconds per block on churned DBs; // see db/rawdb/balcache.go. Older blocks needed by eth/71 peers // or RPC are regenerated on demand via the BALRegenerator. - rawdb.CacheBlockAccessList(header.Hash(), block.BlockAccessList) + balcache.CacheBlockAccessList(header.Hash(), block.BlockAccessList) } e.logger.Trace("Inserted block", "hash", header.Hash(), "number", header.Number) } diff --git a/execution/stagedsync/bal_create.go b/execution/stagedsync/bal_create.go index 62e506f30ce..4bc33c562b7 100644 --- a/execution/stagedsync/bal_create.go +++ b/execution/stagedsync/bal_create.go @@ -8,7 +8,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" @@ -79,7 +79,7 @@ func ProcessBAL(tx kv.TemporalRwTx, h *types.Header, vio *state.VersionedIO, isE // cached BAL here for the validator cross-check. Cache misses are OK — // not every code path has a sidecar (backward block downloader doesn't // carry one). - dbBALBytes, _ := rawdb.CachedBlockAccessList(blockHash) + dbBALBytes, _ := balcache.CachedBlockAccessList(blockHash) if dbBALBytes != nil { dbBAL, err := types.DecodeBlockAccessListBytes(dbBALBytes) if err != nil { diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 0d2b61bd6c0..ab20ad157e0 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/rawdb/rawdbhelpers" "github.com/erigontech/erigon/db/rawdb/rawtemporaldb" dbstate "github.com/erigontech/erigon/db/state" @@ -611,7 +612,7 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m // BALs are cache-only (see db/rawdb/balcache.go). If the engine_newPayload // path cached one for this block (via execmodule.InsertBlocks), pick it up // here for the parallel exec's BAL validation. - data, _ := rawdb.CachedBlockAccessList(b.Hash()) + data, _ := balcache.CachedBlockAccessList(b.Hash()) if len(data) > 0 && !dbg.IgnoreBAL { dbBAL, err = types.DecodeBlockAccessListBytes(data) if err != nil { diff --git a/node/eth/backend.go b/node/eth/backend.go index 18a753c8047..33cfada1fcd 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -62,6 +62,7 @@ import ( "github.com/erigontech/erigon/db/kv/remotedbserver" "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/rawdb/blockio" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/db/snapcfg" @@ -734,7 +735,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger // Expose the on-demand FetchBALs entry point to sync hooks // (engine_block_downloader after a batch, engine_newPayload when // the CL payload doesn't carry a BAL). - rawdb.SetBALSyncFetcher(balDownloader) + balcache.SetBALSyncFetcher(balDownloader) go balDownloader.Run(backend.sentryCtx) } @@ -1040,7 +1041,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger // fall back to re-executing the block when nothing is cached. BAL bytes // are no longer persisted to MDBX (see db/rawdb/balcache.go), so older // blocks needed by peers or RPC must be reconstructed on demand. - rawdb.SetBALRegenerator(execmodule.NewBALRegenerator(execmodule.BALRegeneratorDeps{ + balcache.SetBALRegenerator(execmodule.NewBALRegenerator(execmodule.BALRegeneratorDeps{ DB: backend.chainDB, ChainConfig: chainConfig, Engine: backend.engine, diff --git a/p2p/protocols/eth/handlers.go b/p2p/protocols/eth/handlers.go index efa6568d05e..a7077b884d8 100644 --- a/p2p/protocols/eth/handlers.go +++ b/p2p/protocols/eth/handlers.go @@ -30,6 +30,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/rlp" @@ -209,7 +210,7 @@ func AnswerGetBlockAccessListsQuery(ctx context.Context, db kv.Tx, query GetBloc } // 2-tier lookup: in-memory cache → installed BALRegenerator // (re-executes the block when nothing is cached). No MDBX read. - bal, _ := rawdb.BlockAccessListBytes(ctx, hash, *number) + bal, _ := balcache.BlockAccessListBytes(ctx, hash, *number) if len(bal) == 0 { // We have the block header but no source produced a BAL // (pre-Amsterdam, pruned, or regenerator absent/declined). diff --git a/p2p/protocols/eth/handlers_test.go b/p2p/protocols/eth/handlers_test.go index 71bb9a99a52..42bfe459f44 100644 --- a/p2p/protocols/eth/handlers_test.go +++ b/p2p/protocols/eth/handlers_test.go @@ -9,7 +9,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/db/kv/memdb" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/types" @@ -571,8 +571,8 @@ func (balHeaderReader) Integrity(context.Context) error { panic("not expected") // ethereum/EIPs#11553) for any hash we don't have stored — including unknown // blocks and known blocks with no BAL recorded. func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) { - rawdb.ResetBALCacheForTest() - t.Cleanup(rawdb.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) db := memdb.NewTestDB(t, dbcfg.ChainDB) tx, err := db.BeginRw(context.Background()) if err != nil { @@ -591,7 +591,7 @@ func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) } bal := []byte{0xc3, 0x01, 0x02, 0x03} // short valid RLP payload (non-empty) - rawdb.CacheBlockAccessList(hashKnownWithBAL, bal) + balcache.CacheBlockAccessList(hashKnownWithBAL, bal) query := GetBlockAccessListsPacket{hashKnownWithBAL, hashUnknown, hashKnownNoBAL} result := AnswerGetBlockAccessListsQuery(context.Background(), tx, query, reader) @@ -613,8 +613,8 @@ func TestAnswerGetBlockAccessListsQuery_OrderedResponseWithMissing(t *testing.T) // TestAnswerGetBlockAccessListsQuery_SoftSizeLimit verifies the handler // respects softResponseLimit by truncating the response (not padding). func TestAnswerGetBlockAccessListsQuery_SoftSizeLimit(t *testing.T) { - rawdb.ResetBALCacheForTest() - t.Cleanup(rawdb.ResetBALCacheForTest) + balcache.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) db := memdb.NewTestDB(t, dbcfg.ChainDB) tx, err := db.BeginRw(context.Background()) if err != nil { @@ -642,7 +642,7 @@ func TestAnswerGetBlockAccessListsQuery_SoftSizeLimit(t *testing.T) { h := common.Hash{byte(i + 1)} num := uint64(1000 + i) reader[h] = num - rawdb.CacheBlockAccessList(h, bal) + balcache.CacheBlockAccessList(h, bal) query = append(query, h) } diff --git a/p2p/sentry/sentry_multi_client/bal_downloader.go b/p2p/sentry/sentry_multi_client/bal_downloader.go index 1104fc38348..f6d06c9ab0a 100644 --- a/p2p/sentry/sentry_multi_client/bal_downloader.go +++ b/p2p/sentry/sentry_multi_client/bal_downloader.go @@ -30,7 +30,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/node/gointerfaces/sentryproto" "github.com/erigontech/erigon/node/gointerfaces/typesproto" ) @@ -201,7 +201,7 @@ func (d *BALDownloader) collectMissingBALs(ctx context.Context) ([]missingBAL, e // Cache-only — BALs are not persisted to MDBX. If the cache has // already absorbed this hash (recently produced locally or fetched // from another peer this run), skip the fetch. - if _, ok := rawdb.CachedBlockAccessList(hash); ok { + if _, ok := balcache.CachedBlockAccessList(hash); ok { continue } missing = append(missing, missingBAL{ @@ -290,7 +290,7 @@ func (d *BALDownloader) fetchBatch(ctx context.Context, peer [64]byte, sentryI i if len(payload) == 0 { continue } - rawdb.CacheBlockAccessList(batch[i].hash, payload) + balcache.CacheBlockAccessList(batch[i].hash, payload) stored++ } @@ -304,7 +304,7 @@ func (d *BALDownloader) fetchBatch(ctx context.Context, peer [64]byte, sentryI i } } -// FetchBALs implements rawdb.BALSyncFetcher: picks an eth/71 peer and +// FetchBALs implements balcache.BALSyncFetcher: picks an eth/71 peer and // requests BALs for the given (hash, number, expected) tuples in a single // batched call, caching every hash-verified response. Non-blocking by intent // — failures (no peer, peer declined, timeout) are silently dropped because diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go index b477ece2c7d..dda06a8675a 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go @@ -14,7 +14,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/db/kv/temporal" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/types" @@ -291,9 +291,9 @@ func TestGetBlockAccessLists71_AnswersAndSends(t *testing.T) { hashUnknown := common.Hash{0x02} const knownBlockNum uint64 = 100 bal := []byte{0xc3, 0x01, 0x02, 0x03} // short valid RLP non-empty payload - rawdb.ResetBALCacheForTest() - t.Cleanup(rawdb.ResetBALCacheForTest) - rawdb.CacheBlockAccessList(hashKnown, bal) + balcache.ResetBALCacheForTest() + t.Cleanup(balcache.ResetBALCacheForTest) + balcache.CacheBlockAccessList(hashKnown, bal) if err := rwTx.Commit(); err != nil { t.Fatalf("commit: %v", err) } diff --git a/rpc/jsonrpc/eth_block_access_list.go b/rpc/jsonrpc/eth_block_access_list.go index e516cba161a..7cf70db3038 100644 --- a/rpc/jsonrpc/eth_block_access_list.go +++ b/rpc/jsonrpc/eth_block_access_list.go @@ -20,7 +20,7 @@ import ( "context" "errors" - "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/rpc" "github.com/erigontech/erigon/rpc/ethapi" @@ -66,7 +66,7 @@ func (api *APIImpl) GetBlockAccessList(ctx context.Context, numberOrHash rpc.Blo } } - data, err := rawdb.BlockAccessListBytes(ctx, blockHash, blockNum) + data, err := balcache.BlockAccessListBytes(ctx, blockHash, blockNum) if err != nil { return nil, err } From 52c8ac614c219772ab8de9d9c37a90ba22cecee4 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 08:35:41 +0000 Subject: [PATCH 108/120] =?UTF-8?q?execution/balcache,=20execution/enginea?= =?UTF-8?q?pi,=20execution/exec:=20BAL=20prefetch=20via=20balcache=20?= =?UTF-8?q?=E2=86=92=20cache.StateCache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BlockReadAheader's BAL prefetch was sourcing the access list from MDBX (kv.BlockAccessList table). On the perf-devnet-3 / bench-feeder path the BAL is never persisted to chaindata — it arrives via engine_newPayloadV5 and is held only in the payload. Diagnostic logging confirmed: AddHeaderAndBody fires, warmBody enters with stateCacheWired=true, txs=2 for the test block, but balLen=0 because the BAL was nowhere accessible to read-ahead. Result: no prefetch, EVM hot path paid the full file accessor stack on every cold address. Ports the balcache package from mh/perf-bal-cache as the canonical BAL store on the engine-API path: - execution/balcache/balcache.go — in-memory LRU (100 blocks), Cache/CachedBlockAccessList API, optional BALRegenerator / BALSyncFetcher hooks for fallback paths. - EngineServer.HandleNewPayload writes the BAL bytes into the cache on payload receipt, before any downstream processing. - BlockReadAheader.warmBody reads BAL via balcache.CachedBlockAccessList keyed by block hash, replacing the MDBX read entirely. Bench result (test_account_access[EXTCODESIZE-EXISTING_CONTRACT-30M], perf-devnet-3 v4.0.0 fixture, EXEC3_PARALLEL=true): binary mgas/s block time baseline 12.82 2.4 s L2b only 12.75 2.4 s L2b + codeSize 13.13 2.4 s L2b + codeSize + balcache 61.50 488 ms ← this commit ~4.7x speedup. Cross-client peer band on the same family (cycle-2 survey 2026-05-13): reth 50, geth 55, besu 55, nethermind 70. Erigon at 62 mgas/s now sits between geth and nethermind on a bench that was 2.2 mgas/s in the May 13 cross-client pull. Pprof signature confirms: seg.Getter.nextPos flat dropped from 52 % to 24 %; the dominant decompression cost is gone for BAL-listed addresses because EVM hits cache.StateCache (populated by read-ahead's cache-populating getter, commit cbe9044e52) instead of the file accessor stack. The kv.BlockAccessList table removal + rawdb.WriteBlockAccessListBytes deletion is intentionally NOT in this commit — that belongs to the bal-cache structural refactor on mh/perf-bal-cache, not on the optimisation branch. Co-Authored-By: Claude Opus 4.7 (1M context) --- execution/engineapi/engine_server.go | 11 +++++++++++ execution/exec/blocks_read_ahead.go | 27 ++++++++++++--------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/execution/engineapi/engine_server.go b/execution/engineapi/engine_server.go index fae186810aa..bfc122d36c1 100644 --- a/execution/engineapi/engine_server.go +++ b/execution/engineapi/engine_server.go @@ -43,6 +43,7 @@ import ( "github.com/erigontech/erigon/db/kv/kvcache" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/builder" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/engineapi/engine_block_downloader" @@ -928,6 +929,16 @@ func (e *EngineServer) HandleNewPayload( ) (*engine_types.PayloadStatus, error) { e.engineLogSpamer.RecordRequest() + // Cache the incoming BAL bytes so downstream consumers (read-ahead, + // validation, BAL-driven workers) can fetch from process memory without + // touching MDBX. The bal-cache architecture removes the BAL from MDBX + // on the NewPayload critical path entirely; mirrors the design pattern + // from mh/perf-bal-cache so the eventual refactor is a wiring move, + // not a redesign. + if len(blockAccessListBytes) > 0 { + balcache.CacheBlockAccessList(block.Hash(), blockAccessListBytes) + } + header := block.Header() headerNumber := header.Number.Uint64() headerHash := block.Hash() diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 08c0d47a5c9..1f77e8ace36 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -14,8 +14,8 @@ import ( "github.com/erigontech/erigon/common/length" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" @@ -178,23 +178,20 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var wg errgroup.Group - // If BAL exists in DB, use BAL warming (more complete) + // BAL is sourced from the in-memory balcache (populated by + // EngineServer.HandleNewPayload on receipt). The bal-cache architecture + // removes the BAL from chaindata entirely; cache miss is a clean signal + // that BAL prefetch is not available for this block — fall through to + // per-transaction warming. var bal types.BlockAccessList - if header != nil && db != nil { - tx, err := db.BeginRo(ctx) - if err != nil { - log.Warn("[warmBody] failed to open tx for BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } else { - data, err := tx.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(header.Number.Uint64(), header.Hash())) + if header != nil { + if data, ok := balcache.CachedBlockAccessList(header.Hash()); ok && len(data) > 0 { + decoded, err := types.DecodeBlockAccessListBytes(data) if err != nil { - log.Warn("[warmBody] failed to read BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } else if len(data) > 0 { - bal, err = types.DecodeBlockAccessListBytes(data) - if err != nil { - log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) - } + log.Warn("[warmBody] failed to decode BAL", "blockNum", header.Number.Uint64(), "blockHash", header.Hash(), "err", err) + } else { + bal = decoded } - tx.Rollback() } } From 6427aca1e7b3d725637612f9a2580b1fa6545f97 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 15 May 2026 12:19:14 +0000 Subject: [PATCH 109/120] db/kv, db/rawdb, execution: drop kv.BlockAccessList table and orphan helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BAL bytes are now cache-only (balcache, populated by ComputeBlockContext with on-demand regeneration). After the BAL writers were removed in 9ea90e4307, the MDBX table and its Read/Write/Delete helpers have no producers — drop them outright instead of carrying dead surface. Removed: - kv.BlockAccessList table constant + ChaindataTables entry - rawdb.ReadBlockAccessListBytes / WriteBlockAccessListBytes - BlockAccessList delete in DeleteBody + 2 prune callbacks - PruneTable(kv.BlockAccessList) call in stage_execute - TestBlockAccessListStorage + the BlockAccessList seed rows in TestNoPruneSkipsAllPruneStages Readers (warmBody readAhead) already consult balcache.CachedBlockAccessList; the test helper in engine_server_test writes directly to balcache. Co-Authored-By: Claude Opus 4.7 (1M context) --- db/kv/tables.go | 3 -- db/rawdb/accessors_chain.go | 26 --------- db/rawdb/accessors_chain_test.go | 54 ------------------- .../block_downloader.go | 2 +- execution/exec/blocks_read_ahead.go | 10 ++-- execution/execmodule/balregen.go | 7 +-- execution/execmodule/inserters.go | 2 +- execution/stagedsync/exec3.go | 2 +- execution/stagedsync/no_prune_test.go | 4 +- execution/stagedsync/stage_execute.go | 21 -------- node/eth/backend.go | 2 +- p2p/protocols/eth/handlers.go | 2 +- .../sentry_multi_client_test.go | 2 +- 13 files changed, 16 insertions(+), 121 deletions(-) diff --git a/db/kv/tables.go b/db/kv/tables.go index 10bdb96d15e..713a8b3cb06 100644 --- a/db/kv/tables.go +++ b/db/kv/tables.go @@ -61,8 +61,6 @@ const ( HeaderTD = "HeadersTotalDifficulty" // block_num_u64 + hash -> td (RLP) BlockBody = "BlockBody" // block_num_u64 + hash -> block body - // BlockAccessList stores RLP-encoded block access lists, keyed by block_num_u64 + hash. - BlockAccessList = "BlockAccessList" // Naming: // TxNum - Ethereum canonical transaction number - same across all nodes. @@ -315,7 +313,6 @@ var ChaindataTables = []string{ HeaderNumber, BadHeaderNumber, BlockBody, - BlockAccessList, TxLookup, ConfigTable, DatabaseInfo, diff --git a/db/rawdb/accessors_chain.go b/db/rawdb/accessors_chain.go index 4ea0aefaae2..c6a2ad5e3d2 100644 --- a/db/rawdb/accessors_chain.go +++ b/db/rawdb/accessors_chain.go @@ -595,23 +595,6 @@ func ReadBody(db kv.Getter, hash common.Hash, number uint64) (*types.Body, uint6 return body, bodyForStorage.BaseTxnID.First(), bodyForStorage.TxCount - 2 // 1 system txn in the beginning of block, and 1 at the end } -// ReadBlockAccessListBytes reads the RLP-encoded block access list sidecar for a block. -func ReadBlockAccessListBytes(db kv.Getter, hash common.Hash, number uint64) ([]byte, error) { - data, err := db.GetOne(kv.BlockAccessList, dbutils.BlockBodyKey(number, hash)) - if err != nil { - return nil, err - } - return data, nil -} - -// WriteBlockAccessListBytes stores the RLP-encoded block access list sidecar for a block. -func WriteBlockAccessListBytes(db kv.Putter, hash common.Hash, number uint64, data []byte) error { - if err := db.Put(kv.BlockAccessList, dbutils.BlockBodyKey(number, hash), data); err != nil { - return fmt.Errorf("failed to store block access list: %w", err) - } - return nil -} - func HasSenders(db kv.Getter, hash common.Hash, number uint64) (bool, error) { return db.Has(kv.Senders, dbutils.BlockBodyKey(number, hash)) } @@ -698,9 +681,6 @@ func DeleteBody(db kv.Putter, hash common.Hash, number uint64) { if err := db.Delete(kv.BlockBody, dbutils.BlockBodyKey(number, hash)); err != nil { log.Crit("Failed to delete block body", "err", err) } - if err := db.Delete(kv.BlockAccessList, dbutils.BlockBodyKey(number, hash)); err != nil { - log.Crit("Failed to delete block access list", "err", err) - } } func AppendCanonicalTxNums(tx kv.RwTx, from uint64) (err error) { @@ -898,9 +878,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) (deleted int if err = tx.Delete(kv.BlockBody, kCopy); err != nil { return deleted, err } - if err = tx.Delete(kv.BlockAccessList, kCopy); err != nil { - return deleted, err - } if err = tx.Delete(kv.Headers, kCopy); err != nil { return deleted, err } @@ -949,9 +926,6 @@ func TruncateBlocks(ctx context.Context, tx kv.RwTx, blockFrom uint64) error { if err := tx.Delete(kv.BlockBody, kCopy); err != nil { return err } - if err := tx.Delete(kv.BlockAccessList, kCopy); err != nil { - return err - } if err := tx.Delete(kv.Headers, kCopy); err != nil { return err } diff --git a/db/rawdb/accessors_chain_test.go b/db/rawdb/accessors_chain_test.go index 50f8955fa07..dd3c6483c3d 100644 --- a/db/rawdb/accessors_chain_test.go +++ b/db/rawdb/accessors_chain_test.go @@ -40,7 +40,6 @@ import ( "github.com/erigontech/erigon/execution/execmodule/execmoduletester" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/types" - "github.com/erigontech/erigon/execution/types/accounts" ) func newTestLegacyTx(nonce uint64, to common.Address, value uint256.Int, gasLimit uint64, gasPrice uint256.Int) *types.LegacyTx { @@ -1126,59 +1125,6 @@ func TestBlockWithdrawalsStorage(t *testing.T) { require.Nil(entry) } -func TestBlockAccessListStorage(t *testing.T) { - t.Parallel() - _, tx := memdb.NewTestTx(t) - defer tx.Rollback() - - block := types.NewBlockWithHeader(&types.Header{ - Number: *uint256.NewInt(1), - Extra: []byte("test block"), - UncleHash: empty.UncleHash, - TxHash: empty.RootHash, - ReceiptHash: empty.RootHash, - }) - - data, err := rawdb.ReadBlockAccessListBytes(tx, block.Hash(), block.NumberU64()) - require.NoError(t, err) - require.Empty(t, data) - - nonEmpty := types.BlockAccessList{ - { - Address: accounts.InternAddress(common.HexToAddress("0x00000000000000000000000000000000000000aa")), - }, - } - nonEmptyBytes, err := types.EncodeBlockAccessListBytes(nonEmpty) - require.NoError(t, err) - require.NoError(t, rawdb.WriteBlockAccessListBytes(tx, block.Hash(), block.NumberU64(), nonEmptyBytes)) - - data, err = rawdb.ReadBlockAccessListBytes(tx, block.Hash(), block.NumberU64()) - require.NoError(t, err) - require.Equal(t, nonEmptyBytes, data) - - decoded, err := types.DecodeBlockAccessListBytes(data) - require.NoError(t, err) - require.NoError(t, decoded.Validate()) - require.Equal(t, nonEmpty.Hash(), decoded.Hash()) - - emptyBytes, err := types.EncodeBlockAccessListBytes(nil) - require.NoError(t, err) - require.NoError(t, rawdb.WriteBlockAccessListBytes(tx, block.Hash(), block.NumberU64(), emptyBytes)) - - data, err = rawdb.ReadBlockAccessListBytes(tx, block.Hash(), block.NumberU64()) - require.NoError(t, err) - require.Equal(t, emptyBytes, data) - - decoded, err = types.DecodeBlockAccessListBytes(data) - require.NoError(t, err) - // EIP-7928: Decoding 0xc0 (empty RLP list) should return an initialized empty slice, - // rather than nil, to distinguish a valid empty BAL from a missing/pruned one. - require.NotNil(t, decoded) - require.Empty(t, decoded) - require.NoError(t, decoded.Validate()) - require.Equal(t, empty.BlockAccessListHash, decoded.Hash()) -} - // Tests pre-shanghai body to make sure withdrawals doesn't panic func TestPreShanghaiBodyNoPanicOnWithdrawals(t *testing.T) { t.Parallel() diff --git a/execution/engineapi/engine_block_downloader/block_downloader.go b/execution/engineapi/engine_block_downloader/block_downloader.go index 051f1ea62b1..99db738089e 100644 --- a/execution/engineapi/engine_block_downloader/block_downloader.go +++ b/execution/engineapi/engine_block_downloader/block_downloader.go @@ -29,8 +29,8 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/execmodule" "github.com/erigontech/erigon/execution/execmodule/chainreader" diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 1f77e8ace36..4a2cd808f89 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -178,11 +178,11 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var wg errgroup.Group - // BAL is sourced from the in-memory balcache (populated by - // EngineServer.HandleNewPayload on receipt). The bal-cache architecture - // removes the BAL from chaindata entirely; cache miss is a clean signal - // that BAL prefetch is not available for this block — fall through to - // per-transaction warming. + // BAL source = the in-memory balcache (populated by + // EngineServer.HandleNewPayload on receipt). The chaindata + // kv.BlockAccessList table no longer exists; cache miss is a clean + // signal that BAL prefetch is not available for this block — fall + // through to per-transaction warming below. var bal types.BlockAccessList if header != nil { if data, ok := balcache.CachedBlockAccessList(header.Hash()); ok && len(data) > 0 { diff --git a/execution/execmodule/balregen.go b/execution/execmodule/balregen.go index bd690ca3b7c..ec0039e877a 100644 --- a/execution/execmodule/balregen.go +++ b/execution/execmodule/balregen.go @@ -19,7 +19,8 @@ package execmodule import ( "context" "fmt" - "math/big" + + "github.com/holiman/uint256" "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" @@ -27,13 +28,13 @@ import ( rawdbv3 "github.com/erigontech/erigon/db/kv/rawdbv3" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/execution/chain" - "github.com/erigontech/erigon/execution/vm/evmtypes" "github.com/erigontech/erigon/execution/protocol" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/execution/types/accounts" "github.com/erigontech/erigon/execution/vm" + "github.com/erigontech/erigon/execution/vm/evmtypes" "github.com/erigontech/erigon/rpc/transactions" ) @@ -199,6 +200,6 @@ func (s *chainReaderShim) CurrentSafeHeader() *types.Header func (s *chainReaderShim) GetHeader(hash common.Hash, number uint64) *types.Header { return nil } func (s *chainReaderShim) GetHeaderByNumber(number uint64) *types.Header { return nil } func (s *chainReaderShim) GetHeaderByHash(hash common.Hash) *types.Header { return nil } -func (s *chainReaderShim) GetTd(hash common.Hash, number uint64) *big.Int { return big.NewInt(0) } +func (s *chainReaderShim) GetTd(hash common.Hash, number uint64) *uint256.Int { return uint256.NewInt(0) } func (s *chainReaderShim) FrozenBlocks() uint64 { return 0 } func (s *chainReaderShim) FrozenBorBlocks(align bool) uint64 { return 0 } diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index 4507bb68bb4..654ba3ee50b 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -24,8 +24,8 @@ import ( "github.com/holiman/uint256" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/commitment/commitmentdb" "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/types" diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index ab20ad157e0..30fc537ae6d 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -35,11 +35,11 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/rawdb/rawdbhelpers" "github.com/erigontech/erigon/db/rawdb/rawtemporaldb" dbstate "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/exec" "github.com/erigontech/erigon/execution/protocol" diff --git a/execution/stagedsync/no_prune_test.go b/execution/stagedsync/no_prune_test.go index f66f7475f85..e1b9d20bed8 100644 --- a/execution/stagedsync/no_prune_test.go +++ b/execution/stagedsync/no_prune_test.go @@ -51,8 +51,6 @@ func TestNoPruneSkipsAllPruneStages(t *testing.T) { seeds := []seedRow{ {kv.ChangeSets3, "k1", "v1"}, {kv.ChangeSets3, "k2", "v2"}, - {kv.BlockAccessList, "b1", "ba1"}, - {kv.BlockAccessList, "b2", "ba2"}, {kv.TxLookup, "t1", "tl1"}, {kv.BorWitnesses, "w1", "wit1"}, } @@ -70,7 +68,7 @@ func TestNoPruneSkipsAllPruneStages(t *testing.T) { } return n } - tracked := []string{kv.ChangeSets3, kv.BlockAccessList, kv.TxLookup, kv.BorWitnesses} + tracked := []string{kv.ChangeSets3, kv.TxLookup, kv.BorWitnesses} pre := map[string]int{} for _, table := range tracked { pre[table] = countRows(t, table) diff --git a/execution/stagedsync/stage_execute.go b/execution/stagedsync/stage_execute.go index dccf76087fa..3f3c7a8efe9 100644 --- a/execution/stagedsync/stage_execute.go +++ b/execution/stagedsync/stage_execute.go @@ -528,27 +528,6 @@ func PruneExecutionStage(ctx context.Context, s *PruneState, tx kv.RwTx, cfg Exe } } - if s.ForwardProgress > cfg.syncCfg.MaxReorgDepth { - pruneBalLimit := 10_000 - pruneTimeout := quickPruneTimeout - if s.CurrentSyncCycle.IsInitialCycle { - pruneBalLimit = math.MaxInt - pruneTimeout = time.Hour - } - if err := rawdb.PruneTable( - tx, - kv.BlockAccessList, - s.ForwardProgress-cfg.syncCfg.MaxReorgDepth, - ctx, - pruneBalLimit, - pruneTimeout, - logger, - s.LogPrefix(), - ); err != nil { - return err - } - } - agg := cfg.db.(state.HasAgg).Agg().(*state.Aggregator) mxExecStepsInDB.Set(rawdbhelpers.IdxStepsCountV3(tx, agg.StepSize()) * 100) diff --git a/node/eth/backend.go b/node/eth/backend.go index 33cfada1fcd..407ff7672df 100644 --- a/node/eth/backend.go +++ b/node/eth/backend.go @@ -62,7 +62,6 @@ import ( "github.com/erigontech/erigon/db/kv/remotedbserver" "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/rawdb/blockio" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/db/snapcfg" @@ -72,6 +71,7 @@ import ( "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/diagnostics/diaglib" "github.com/erigontech/erigon/diagnostics/mem" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/builder" "github.com/erigontech/erigon/execution/chain" chainspec "github.com/erigontech/erigon/execution/chain/spec" diff --git a/p2p/protocols/eth/handlers.go b/p2p/protocols/eth/handlers.go index a7077b884d8..92ead0bd5cd 100644 --- a/p2p/protocols/eth/handlers.go +++ b/p2p/protocols/eth/handlers.go @@ -30,8 +30,8 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/types" diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go index dda06a8675a..e7759d72304 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client_test.go @@ -14,8 +14,8 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/db/kv/temporal" - "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/node/direct" From cd839e1d19a8c1d1a882d8e90e382571c4574bf1 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 28 Apr 2026 21:47:02 +0000 Subject: [PATCH 110/120] rpc/jsonrpc: debug_getRawBlockAccessList returns balcache bytes (porting from the MDBX-era diagnostic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds debug_getRawBlockAccessList JSON-RPC method — returns the RLP-encoded BlockAccessList bytes this node has stored for a block (exactly what the eth/71 server-side handler returns to peers). Originally introduced as part of the dispatch-bug diagnostic tooling (b8bd26caa8) which also added cmd/bal-scan and cmd/bal-test. The cmd tools were tightly coupled to the kv.BlockAccessList MDBX table (scan / dump / delete / compare via cursor) — that table is removed in the prior commit of this PR, so the cmd tools have no surviving port. The RPC method does: it switches its read from rawdb.ReadBlockAccessListBytes to balcache.BlockAccessListBytes (cache hit, regenerator fallback). Co-Authored-By: Claude Opus 4.7 (1M context) --- rpc/jsonrpc/debug_api.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rpc/jsonrpc/debug_api.go b/rpc/jsonrpc/debug_api.go index 9780ec91a02..050df599189 100644 --- a/rpc/jsonrpc/debug_api.go +++ b/rpc/jsonrpc/debug_api.go @@ -30,6 +30,7 @@ import ( "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/rawdb" + "github.com/erigontech/erigon/execution/balcache" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/stagedsync/stages" "github.com/erigontech/erigon/execution/state" @@ -63,6 +64,7 @@ type PrivateDebugAPI interface { AccountAt(ctx context.Context, blockHash common.Hash, txIndex uint64, account common.Address) (*AccountResult, error) GetRawHeader(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) GetRawBlock(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) + GetRawBlockAccessList(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) GetBadBlocks(ctx context.Context) ([]map[string]any, error) GetRawTransaction(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) @@ -671,6 +673,31 @@ func (api *DebugAPIImpl) GetRawBlock(ctx context.Context, blockNrOrHash rpc.Bloc return rlp.EncodeToBytes(block) } +// GetRawBlockAccessList implements debug_getRawBlockAccessList — returns the +// raw RLP-encoded BlockAccessList bytes (EIP-7928) that this node has stored +// for the given block. Returns nil if no BAL is recorded (pre-Amsterdam, or +// post-Amsterdam but not yet downloaded). The bytes returned are exactly what +// the server-side eth/71 GetBlockAccessLists handler would send to a peer. +func (api *DebugAPIImpl) GetRawBlockAccessList(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + tx, err := api.db.BeginTemporalRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + n, h, _, err := rpchelper.GetBlockNumber(ctx, blockNrOrHash, tx, api._blockReader, api.filters) + if err != nil { + if errors.As(err, &rpc.BlockNotFoundErr{}) { + return nil, nil + } + return nil, err + } + bal, err := balcache.BlockAccessListBytes(ctx, h, n) + if err != nil { + return nil, fmt.Errorf("read block access list: %w", err) + } + return bal, nil +} + // GetRawReceipts implements debug_getRawReceipts - retrieves and returns an array of EIP-2718 binary-encoded receipts of a single block func (api *DebugAPIImpl) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) { tx, err := api.db.BeginTemporalRo(ctx) From 2a4119265a09133df4397d86b84fb358c17bae03 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 16:54:16 +0000 Subject: [PATCH 111/120] execution/cache, state, stagedsync: txNum/epoch cache invalidation; remove the schedule-time ValidateAndPrepare purge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state cache carried a per-domain blockHash and was scrubbed by ValidateAndPrepare before every block. In the parallel executor that call sits in processRequest — a *schedule* step, not an apply step — copied from the serial path (#18868). With a 32-deep pipeline and heavy retry traffic the single blockHash almost never equals the next call's parentHash, so it took the wipe branch ~100% of the time: measured storage-cache purge_rate ~100%, hit ~35% during catch-up, the cache wiped every block. Make the cache what it should be — a SharedDomains implementation detail, populated only at flush (committed, fork-agnostic state) and invalidated only on unwind. Coherence is now txNum/epoch based, no block awareness and no diffset: - Each GenericCache entry carries (txNum, epoch). A read is valid iff it was written in the current epoch OR its txNum is at/below unwindFloor (predates every unwind). - Unwind(txNum) bumps the epoch and lowers the floor — O(1), no scan. Stale entries are dropped lazily on their next read. txNum slots are reused across forks, so the epoch (not the txNum) tells a dead fork's write from the live fork's at the same txNum. - CodeCache clears its small mutable addr layers on unwind; immutable content-addressed code is kept. Wiring: FlushWithCallback delivers txNum (cache stamps it; branchCache derives step); read-population and read-ahead stamp the step's txNum upper bound. The three exec-flow ValidateAndPrepare calls are removed; the unwind path calls stateCache.Unwind(txNum) unconditionally (diffset-free, matching the overlay's maxtx prune — diffsets aren't generated below the reorg window, so the old changeSet-gated cache revert left a stale gap). RevertWithDiffset/blockHash/ClearWithHash and the fork-validation cache scrub are removed. (DB-level diffset retirement is a follow-on.) Co-Authored-By: Claude Opus 4.8 (1M context) --- db/kv/kv_interface.go | 2 +- db/state/execctx/domain_shared.go | 20 +- db/state/temporal_mem_batch.go | 6 +- execution/cache/cache.go | 24 +- execution/cache/cache_test.go | 501 +++++-------------------- execution/cache/code_cache.go | 66 +--- execution/cache/generic_cache.go | 122 +++--- execution/cache/state_cache.go | 79 +--- execution/exec/blocks_read_ahead.go | 9 +- execution/execmodule/exec_module.go | 10 +- execution/stagedsync/exec3.go | 7 - execution/stagedsync/exec3_parallel.go | 18 +- execution/stagedsync/exec3_serial.go | 6 +- execution/stagedsync/stage_execute.go | 17 +- execution/vm/contract.go | 3 +- 15 files changed, 243 insertions(+), 647 deletions(-) diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e757a32effd..7aee2feaee0 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -530,7 +530,7 @@ type TemporalMemBatch interface { HasPrefixInRAM(domain Domain, prefix []byte) bool SizeEstimate() uint64 Flush(ctx context.Context, tx RwTx) error - FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, step Step)) error + FlushWithCallback(ctx context.Context, tx RwTx, cb func(domain Domain, k []byte, v []byte, txNum uint64)) error Close() PutForkable(id ForkableId, num Num, v []byte) error DiscardWrites(domain Domain) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 3f9ebd930b5..a184e56219c 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -789,14 +789,14 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { // never per-write — so they mirror committed, fork-agnostic state: // CommitmentDomain → BranchCache, Accounts/Storage/Code → StateCache. if sd.branchCache != nil || sd.stateCache != nil { - if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, step kv.Step) { + if err := sd.mem.FlushWithCallback(ctx, tx, func(domain kv.Domain, k []byte, v []byte, txNum uint64) { switch domain { case kv.CommitmentDomain: if sd.branchCache != nil { if len(v) == 0 { sd.branchCache.Invalidate(k) } else { - sd.branchCache.Put(k, v, uint64(step), "sd.Flush") + sd.branchCache.Put(k, v, txNum/sd.StepSize(), "sd.Flush") } } case kv.AccountsDomain: @@ -806,7 +806,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { sd.stateCache.Delete(kv.CodeDomain, k) sd.stateCache.DeleteAddrCodeHash(k) } else { - sd.stateCache.Put(kv.AccountsDomain, k, v) + sd.stateCache.Put(kv.AccountsDomain, k, v, txNum) sd.stateCache.DeleteAddrCodeHash(k) } } @@ -815,7 +815,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.stateCache.Delete(kv.StorageDomain, k) } else { - sd.stateCache.Put(kv.StorageDomain, k, v) + sd.stateCache.Put(kv.StorageDomain, k, v, txNum) } } case kv.CodeDomain: @@ -823,7 +823,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.stateCache.Delete(kv.CodeDomain, k) } else { - sd.stateCache.Put(kv.CodeDomain, k, v) + sd.stateCache.Put(kv.CodeDomain, k, v, txNum) } } } @@ -1029,12 +1029,18 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) return nil, 0, fmt.Errorf("storage %x read error: %w", k, err) } - // Populate state cache on successful storage read + // Populate state cache on successful read. Stamp with an upper bound on the + // value's write txNum (the read only gives us the file/step it came from): + // the last txNum of that step. An unwind below this bound can't leave the + // value stale, so frozen-step reads stay warm across unwinds while + // recent-step reads are dropped. (CodeCache addr layers are cleared wholesale + // on unwind, so PutCodeWithHash needs no txNum.) if sd.stateCache != nil { if domain == kv.CodeDomain && len(codeEthHash) > 0 && len(v) > 0 { sd.stateCache.PutCodeWithHash(k, v, codeEthHash) } else { - sd.stateCache.Put(domain, k, v) + readTxNum := (uint64(step)+1)*sd.StepSize() - 1 + sd.stateCache.Put(domain, k, v, readTxNum) } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index fa67b0bd1f5..d68afa04828 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -779,7 +779,7 @@ func (sd *TemporalMemBatch) Merge(o kv.TemporalMemBatch) error { // flush, so they hold committed, fork-agnostic state. func (sd *TemporalMemBatch) FlushWithCallback( ctx context.Context, tx kv.RwTx, - cb func(domain kv.Domain, k []byte, v []byte, step kv.Step), + cb func(domain kv.Domain, k []byte, v []byte, txNum uint64), ) error { sd.latestStateLock.Lock() defer sd.latestStateLock.Unlock() @@ -790,7 +790,7 @@ func (sd *TemporalMemBatch) FlushWithCallback( continue } latest := history[len(history)-1] - cb(kv.Domain(d), []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.Domain(d), []byte(keyStr), latest.data, latest.txNum) } } // StorageDomain entries live in sd.storage (a btree), not sd.domains. @@ -799,7 +799,7 @@ func (sd *TemporalMemBatch) FlushWithCallback( return true } latest := history[len(history)-1] - cb(kv.StorageDomain, []byte(keyStr), latest.data, kv.Step(latest.txNum/sd.stepSize)) + cb(kv.StorageDomain, []byte(keyStr), latest.data, latest.txNum) return true }) diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 170ab31dee7..2171137a0ca 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -16,16 +16,15 @@ package cache -import "github.com/erigontech/erigon/common" - // Cache is the interface for domain caches. // Implementations: GenericCache (for Account/Storage), CodeCache (for Code). type Cache interface { // Get retrieves data for the given key. Get(key []byte) ([]byte, bool) - // Put stores data for the given key. - Put(key []byte, value []byte) + // Put stores data for the given key, stamped with the txNum the value + // reflects (used for txNum/epoch unwind invalidation). + Put(key []byte, value []byte, txNum uint64) // Delete removes the data for the given key. Delete(key []byte) @@ -33,18 +32,11 @@ type Cache interface { // Clear removes all mutable entries from the cache. Clear() - // GetBlockHash returns the hash of the last block processed. - GetBlockHash() common.Hash - - // SetBlockHash sets the hash of the current block being processed. - SetBlockHash(hash common.Hash) - - // ValidateAndPrepare checks if parentHash matches the cache's blockHash. - // If mismatch, clears mutable caches. Returns true if valid, false if cleared. - ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool - - // ClearWithHash clears the cache and sets the block hash. - ClearWithHash(hash common.Hash) + // Unwind invalidates entries that reflect state above unwindToTxNum on a + // now-dead fork. Diffset-free: GenericCache bumps an epoch and lowers a + // floor (lazy evict on read); CodeCache clears its small mutable addr + // layers. Immutable content-addressed code layers are untouched. + Unwind(unwindToTxNum uint64) // Len returns the number of entries in the cache. Len() int diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 3160d45d216..ca83eb8b5a9 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -53,13 +53,6 @@ func makeValue(i int) []byte { // DomainCache Tests // ============================================================================= -func TestDomainCache_NewDomainCache(t *testing.T) { - c := NewDomainCache(100) - require.NotNil(t, c) - assert.Equal(t, 0, c.Len()) - assert.Equal(t, common.Hash{}, c.GetBlockHash()) -} - func TestDomainCache_NewWithByteCapacity(t *testing.T) { c := NewDomainCache(1 * datasize.MB) // 1MB require.NotNil(t, c) @@ -80,7 +73,7 @@ func TestDomainCache_GetPut(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(addr, value) + c.Put(addr, value, 0) v, ok = c.Get(addr) assert.True(t, ok) assert.Equal(t, value, v) @@ -94,10 +87,10 @@ func TestDomainCache_PutUpdateValue(t *testing.T) { value1 := []byte{1, 2, 3, 4, 5, 6, 7, 8} // 8 bytes value2 := []byte{9, 10, 11} // 3 bytes - c.Put(addr, value1) + c.Put(addr, value1, 0) // Update with different value - c.Put(addr, value2) + c.Put(addr, value2, 0) v, ok := c.Get(addr) assert.True(t, ok) @@ -111,11 +104,11 @@ func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { // Two entries take 94 bytes; cap at 100 leaves no room for a third. c := NewDomainCacheMode(100, ModeNoOp) - c.Put(makeAddr(1), makeValue(1)) - c.Put(makeAddr(2), makeValue(2)) + c.Put(makeAddr(1), makeValue(1), 0) + c.Put(makeAddr(2), makeValue(2), 0) assert.Equal(t, 2, c.Len()) - c.Put(makeAddr(3), makeValue(3)) + c.Put(makeAddr(3), makeValue(3), 0) assert.Equal(t, 2, c.Len()) _, ok := c.Get(makeAddr(3)) @@ -123,7 +116,7 @@ func TestDomainCache_PutCapacityLimit_NoOpMode(t *testing.T) { // Updating an existing key always succeeds in either mode. newValue := []byte{100, 101, 102} - c.Put(makeAddr(1), newValue) + c.Put(makeAddr(1), newValue, 0) v, ok := c.Get(makeAddr(1)) assert.True(t, ok) assert.Equal(t, newValue, v) @@ -142,7 +135,7 @@ func TestDomainCache_PutEvictsWhenFull_EvictMode(t *testing.T) { } for i := 1; i <= 64; i++ { - c.Put(makeAddr(i), makeValue(i)) + c.Put(makeAddr(i), makeValue(i), 0) } // The newest entry must still be findable. v, ok := c.Get(makeAddr(64)) @@ -163,7 +156,7 @@ func TestDomainCache_Delete(t *testing.T) { c := NewDomainCache(100) addr := makeAddr(1) - c.Put(addr, makeValue(1)) + c.Put(addr, makeValue(1), 0) assert.Equal(t, 1, c.Len()) c.Delete(addr) @@ -176,89 +169,19 @@ func TestDomainCache_Delete(t *testing.T) { func TestDomainCache_Clear(t *testing.T) { c := NewDomainCache(100) - c.Put(makeAddr(1), makeValue(1)) - c.Put(makeAddr(2), makeValue(2)) + c.Put(makeAddr(1), makeValue(1), 0) + c.Put(makeAddr(2), makeValue(2), 0) assert.Equal(t, 2, c.Len()) c.Clear() assert.Equal(t, 0, c.Len()) } -func TestDomainCache_BlockHash(t *testing.T) { - c := NewDomainCache(100) - - assert.Equal(t, common.Hash{}, c.GetBlockHash()) - - hash := makeHash(1) - c.SetBlockHash(hash) - assert.Equal(t, hash, c.GetBlockHash()) -} - -func TestDomainCache_ValidateAndPrepare_EmptyCache(t *testing.T) { - c := NewDomainCache(100) - - parentHash := makeHash(1) - incomingHash := makeHash(2) - - // Empty cache should always be valid - valid := c.ValidateAndPrepare(parentHash, incomingHash) - assert.True(t, valid) - assert.Equal(t, incomingHash, c.GetBlockHash()) -} - -func TestDomainCache_ValidateAndPrepare_MatchingParent(t *testing.T) { - c := NewDomainCache(100) - - block1 := makeHash(1) - block2 := makeHash(2) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeValue(1)) - - // Parent matches current block hash - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - assert.Equal(t, block2, c.GetBlockHash()) - // Data should be preserved - assert.Equal(t, 1, c.Len()) -} - -func TestDomainCache_ValidateAndPrepare_Mismatch(t *testing.T) { - c := NewDomainCache(100) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeValue(1)) - - // Parent doesn't match - should clear - valid := c.ValidateAndPrepare(block2, block3) - assert.False(t, valid) - assert.Equal(t, block3, c.GetBlockHash()) - // Data should be cleared - assert.Equal(t, 0, c.Len()) -} - -func TestDomainCache_ClearWithHash(t *testing.T) { - c := NewDomainCache(100) - - c.Put(makeAddr(1), makeValue(1)) - c.SetBlockHash(makeHash(1)) - - newHash := makeHash(2) - c.ClearWithHash(newHash) - - assert.Equal(t, 0, c.Len()) - assert.Equal(t, newHash, c.GetBlockHash()) -} - func TestDomainCache_PrintStatsAndReset(t *testing.T) { c := NewDomainCache(100) // Generate some hits and misses - c.Put(makeAddr(1), makeValue(1)) + c.Put(makeAddr(1), makeValue(1), 0) c.Get(makeAddr(1)) // hit c.Get(makeAddr(1)) // hit c.Get(makeAddr(2)) // miss @@ -284,14 +207,6 @@ func TestDomainCache_ImplementsInterface(t *testing.T) { // CodeCache Tests // ============================================================================= -func TestCodeCache_NewCodeCache(t *testing.T) { - c := NewCodeCache(100, 200) - require.NotNil(t, c) - assert.Equal(t, 0, c.Len()) - assert.Equal(t, 0, c.CodeLen()) - assert.Equal(t, common.Hash{}, c.GetBlockHash()) -} - func TestCodeCache_NewDefaultCodeCache(t *testing.T) { c := NewDefaultCodeCache() require.NotNil(t, c) @@ -311,7 +226,7 @@ func TestCodeCache_GetPut(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(addr, code) + c.Put(addr, code, 0) v, ok = c.Get(addr) assert.True(t, ok) assert.Equal(t, code, v) @@ -323,7 +238,7 @@ func TestCodeCache_PutEmptyCode(t *testing.T) { c := NewCodeCache(100, 200) addr := makeAddr(1) - c.Put(addr, []byte{}) + c.Put(addr, []byte{}, 0) // Should not store empty code assert.Equal(t, 0, c.Len()) @@ -339,9 +254,9 @@ func TestCodeCache_CodeDeduplication(t *testing.T) { addr3 := makeAddr(3) // Three addresses with same code - c.Put(addr1, code) - c.Put(addr2, code) - c.Put(addr3, code) + c.Put(addr1, code, 0) + c.Put(addr2, code, 0) + c.Put(addr3, code, 0) // Should have 3 address mappings but only 1 code entry assert.Equal(t, 3, c.Len()) @@ -374,7 +289,7 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { c := NewCodeCache(1024*1024, 1024*28) // 1MB code, ~1024 addr LRU entries for i := 0; i < 1100; i++ { - c.Put(wideAddr(i), wideCode(i)) + c.Put(wideAddr(i), wideCode(i), 0) } // Len should be exactly the LRU cap (1024), not silently truncated to 0. @@ -396,7 +311,7 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { assert.Equal(t, 1100, c.CodeLen()) // Updating an existing addr re-writes the entry (LRU promotes to MRU). - c.Put(wideAddr(1099), wideCode(4242)) + c.Put(wideAddr(1099), wideCode(4242), 0) v, ok = c.Get(wideAddr(1099)) assert.True(t, ok) assert.Equal(t, wideCode(4242), v) @@ -408,12 +323,12 @@ func TestCodeCache_CodeCapacityLimit(t *testing.T) { c := NewCodeCache(25, 1024*1024) // 25 bytes code, 1MB addr // Fill code capacity - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) + c.Put(makeAddr(1), makeCode(1), 0) + c.Put(makeAddr(2), makeCode(2), 0) assert.Equal(t, 2, c.CodeLen()) // Try to add more code - addr mapping added, but code not stored - c.Put(makeAddr(3), makeCode(3)) + c.Put(makeAddr(3), makeCode(3), 0) assert.Equal(t, 3, c.Len()) // addr mapping added assert.Equal(t, 2, c.CodeLen()) // code not added (at capacity) @@ -427,7 +342,7 @@ func TestCodeCache_Delete(t *testing.T) { addr := makeAddr(1) code := makeCode(1) - c.Put(addr, code) + c.Put(addr, code, 0) c.Delete(addr) assert.Equal(t, 0, c.Len()) @@ -441,8 +356,8 @@ func TestCodeCache_Delete(t *testing.T) { func TestCodeCache_Clear(t *testing.T) { c := NewCodeCache(100, 200) - c.Put(makeAddr(1), makeCode(1)) - c.Put(makeAddr(2), makeCode(2)) + c.Put(makeAddr(1), makeCode(1), 0) + c.Put(makeAddr(2), makeCode(2), 0) c.Clear() assert.Equal(t, 0, c.Len()) @@ -450,81 +365,10 @@ func TestCodeCache_Clear(t *testing.T) { assert.Equal(t, 2, c.CodeLen()) } -func TestCodeCache_BlockHash(t *testing.T) { - c := NewCodeCache(100, 200) - - assert.Equal(t, common.Hash{}, c.GetBlockHash()) - - hash := makeHash(1) - c.SetBlockHash(hash) - assert.Equal(t, hash, c.GetBlockHash()) -} - -func TestCodeCache_ValidateAndPrepare_EmptyCache(t *testing.T) { - c := NewCodeCache(100, 200) - - parentHash := makeHash(1) - incomingHash := makeHash(2) - - valid := c.ValidateAndPrepare(parentHash, incomingHash) - assert.True(t, valid) - assert.Equal(t, incomingHash, c.GetBlockHash()) -} - -func TestCodeCache_ValidateAndPrepare_MatchingParent(t *testing.T) { - c := NewCodeCache(100, 200) - - block1 := makeHash(1) - block2 := makeHash(2) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeCode(1)) - - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - assert.Equal(t, block2, c.GetBlockHash()) - // Addr mapping should be preserved - assert.Equal(t, 1, c.Len()) -} - -func TestCodeCache_ValidateAndPrepare_Mismatch(t *testing.T) { - c := NewCodeCache(100, 200) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - c.SetBlockHash(block1) - c.Put(makeAddr(1), makeCode(1)) - - valid := c.ValidateAndPrepare(block2, block3) - assert.False(t, valid) - assert.Equal(t, block3, c.GetBlockHash()) - // Addr mapping should be cleared - assert.Equal(t, 0, c.Len()) - // Code should still exist (immutable) - assert.Equal(t, 1, c.CodeLen()) -} - -func TestCodeCache_ClearWithHash(t *testing.T) { - c := NewCodeCache(100, 200) - - c.Put(makeAddr(1), makeCode(1)) - c.SetBlockHash(makeHash(1)) - - newHash := makeHash(2) - c.ClearWithHash(newHash) - - assert.Equal(t, 0, c.Len()) - assert.Equal(t, newHash, c.GetBlockHash()) - // Code should still exist - assert.Equal(t, 1, c.CodeLen()) -} - func TestCodeCache_PrintStatsAndReset(t *testing.T) { c := NewCodeCache(100, 200) - c.Put(makeAddr(1), makeCode(1)) + c.Put(makeAddr(1), makeCode(1), 0) c.Get(makeAddr(1)) // hit c.Get(makeAddr(2)) // miss @@ -544,7 +388,7 @@ func TestCodeCache_GetMissingCode(t *testing.T) { // Manually set addr mapping without code (simulates capacity limit scenario) addr := makeAddr(1) code := makeCode(1) - c.Put(addr, code) + c.Put(addr, code, 0) // Clear the code cache but keep addr mapping c.hashToCode.Clear() @@ -598,7 +442,7 @@ func TestStateCache_GetPut_Account(t *testing.T) { assert.Nil(t, v) // Put and Get - c.Put(kv.AccountsDomain, addr, value) + c.Put(kv.AccountsDomain, addr, value, 0) v, ok = c.Get(kv.AccountsDomain, addr) assert.True(t, ok) assert.Equal(t, value, v) @@ -612,7 +456,7 @@ func TestStateCache_GetPut_Storage(t *testing.T) { key[51] = 1 value := makeValue(1) - c.Put(kv.StorageDomain, key, value) + c.Put(kv.StorageDomain, key, value, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok) assert.Equal(t, value, v) @@ -624,7 +468,7 @@ func TestStateCache_GetPut_Code(t *testing.T) { addr := makeAddr(1) code := makeCode(1) - c.Put(kv.CodeDomain, addr, code) + c.Put(kv.CodeDomain, addr, code, 0) v, ok := c.Get(kv.CodeDomain, addr) assert.True(t, ok) assert.Equal(t, code, v) @@ -634,7 +478,7 @@ func TestStateCache_GetPut_UnsupportedDomain(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) // ReceiptDomain is not supported - c.Put(kv.ReceiptDomain, makeAddr(1), makeValue(1)) + c.Put(kv.ReceiptDomain, makeAddr(1), makeValue(1), 0) v, ok := c.Get(kv.ReceiptDomain, makeAddr(1)) assert.False(t, ok) assert.Nil(t, v) @@ -644,7 +488,7 @@ func TestStateCache_Delete(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) addr := makeAddr(1) - c.Put(kv.AccountsDomain, addr, makeValue(1)) + c.Put(kv.AccountsDomain, addr, makeValue(1), 0) c.Delete(kv.AccountsDomain, addr) _, ok := c.Get(kv.AccountsDomain, addr) @@ -661,7 +505,7 @@ func TestStateCache_PutEmpty_ThenGet_IsCacheHit(t *testing.T) { key[0] = 0x1d key[51] = 0xa2 - c.Put(kv.StorageDomain, key, nil) + c.Put(kv.StorageDomain, key, nil, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok, "Get after Put(nil) must be a cache hit, not a miss") @@ -676,7 +520,7 @@ func TestStateCache_PutEmptySlice_ThenGet_IsCacheHit(t *testing.T) { key[0] = 0x1d key[51] = 0xa2 - c.Put(kv.StorageDomain, key, []byte{}) + c.Put(kv.StorageDomain, key, []byte{}, 0) v, ok := c.Get(kv.StorageDomain, key) assert.True(t, ok, "Get after Put([]byte{}) must be a cache hit") @@ -693,9 +537,9 @@ func TestStateCache_Delete_UnsupportedDomain(t *testing.T) { func TestStateCache_Clear(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) - c.Put(kv.CodeDomain, makeAddr(3), makeCode(3)) + c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1), 0) + c.Put(kv.StorageDomain, makeAddr(2), makeValue(2), 0) + c.Put(kv.CodeDomain, makeAddr(3), makeCode(3), 0) c.Clear() @@ -708,51 +552,6 @@ func TestStateCache_Clear(t *testing.T) { assert.False(t, ok3) } -func TestStateCache_ValidateAndPrepare_AllValid(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - block1 := makeHash(1) - block2 := makeHash(2) - - // Set same block hash on all caches - c.GetCache(kv.AccountsDomain).SetBlockHash(block1) - c.GetCache(kv.StorageDomain).SetBlockHash(block1) - c.GetCache(kv.CodeDomain).SetBlockHash(block1) - - valid := c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) -} - -func TestStateCache_ValidateAndPrepare_SomeMismatch(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - // Set different block hashes - c.GetCache(kv.AccountsDomain).SetBlockHash(block1) - c.GetCache(kv.StorageDomain).SetBlockHash(block2) // mismatch - c.GetCache(kv.CodeDomain).SetBlockHash(block1) - - valid := c.ValidateAndPrepare(block1, block3) - assert.False(t, valid) -} - -func TestStateCache_ClearWithHash(t *testing.T) { - c := NewStateCache(100, 100, 100, 100, 100) - - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) - - newHash := makeHash(5) - c.ClearWithHash(newHash) - - assert.Equal(t, newHash, c.GetCache(kv.AccountsDomain).GetBlockHash()) - assert.Equal(t, newHash, c.GetCache(kv.StorageDomain).GetBlockHash()) - assert.Equal(t, newHash, c.GetCache(kv.CodeDomain).GetBlockHash()) -} - func TestStateCache_GetCache_OutOfBounds(t *testing.T) { c := NewStateCache(100, 100, 100, 100, 100) @@ -776,7 +575,7 @@ func TestDomainCache_ConcurrentAccess(t *testing.T) { // Writer goroutine go func() { for i := 0; i < 100; i++ { - c.Put(makeAddr(i), makeValue(i)) + c.Put(makeAddr(i), makeValue(i), 0) } done <- true }() @@ -801,7 +600,7 @@ func TestCodeCache_ConcurrentAccess(t *testing.T) { // Writer goroutine go func() { for i := 0; i < 100; i++ { - c.Put(makeAddr(i), makeCode(i)) + c.Put(makeAddr(i), makeCode(i), 0) } done <- true }() @@ -830,9 +629,9 @@ func TestStateCache_DomainIsolation(t *testing.T) { storageData := []byte("storage") codeData := []byte{0x60, 0x00, 0x60, 0x00} // valid code - c.Put(kv.AccountsDomain, addr, accountData) - c.Put(kv.StorageDomain, addr, storageData) - c.Put(kv.CodeDomain, addr, codeData) + c.Put(kv.AccountsDomain, addr, accountData, 0) + c.Put(kv.StorageDomain, addr, storageData, 0) + c.Put(kv.CodeDomain, addr, codeData, 0) v1, ok1 := c.Get(kv.AccountsDomain, addr) v2, ok2 := c.Get(kv.StorageDomain, addr) @@ -851,50 +650,13 @@ func TestStateCache_DomainIsolation(t *testing.T) { // Block Continuity Tests // ============================================================================= -func TestBlockContinuity_Sequential(t *testing.T) { - c := NewDefaultStateCache() - - block0 := common.Hash{} - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - - // Block 1 (parent = empty) - valid := c.ValidateAndPrepare(block0, block1) - assert.True(t, valid) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - - // Block 2 (parent = block1) - valid = c.ValidateAndPrepare(block1, block2) - assert.True(t, valid) - // Data from block 1 should still be there - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.True(t, ok) - - // Block 3 (parent = block2) - valid = c.ValidateAndPrepare(block2, block3) - assert.True(t, valid) -} - -func TestBlockContinuity_Reorg(t *testing.T) { - c := NewDefaultStateCache() - - block1 := makeHash(1) - block2 := makeHash(2) - block2prime := makeHash(22) // fork - - // Build on block 1 - c.ValidateAndPrepare(common.Hash{}, block1) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - - // Continue to block 2 - c.ValidateAndPrepare(block1, block2) - - // Reorg: new block with different parent - valid := c.ValidateAndPrepare(block1, block2prime) - assert.False(t, valid) // Should detect mismatch -} - +// Fork-validation (engine_newPayload) of a block building on the canonical tip +// must NOT purge the hot cache when that block is subsequently applied +// canonically. Regression for the tip purge_rate bug: fork-validation advancing +// blockHash to the speculative block made the canonical apply mismatch & purge. +// Fork-validation of a block on a DIFFERENT parent (reorg proposal) must still +// purge, since cache-as-of-canonical-tip would serve incoherent reads, but it +// must not advance blockHash (canonical continues cleanly afterward). // ============================================================================= // RevertWithDiffset Tests // ============================================================================= @@ -916,134 +678,75 @@ func makeDiffKey(baseKey []byte, step uint64) string { return string(k) } -func TestRevertWithDiffset_SurgicalEviction(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(3) - blockTarget := makeHash(1) - - // Simulate cache state at the tip block. - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) // touched by unwind - c.Put(kv.AccountsDomain, makeAddr(2), makeValue(2)) // untouched - c.Put(kv.StorageDomain, makeAddr(3), makeValue(3)) // touched by unwind - c.Put(kv.StorageDomain, makeAddr(4), makeValue(4)) // untouched - - // Build diffset: addr(1) changed in accounts, addr(3) changed in storage. - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(1), 0), Value: makeValue(10)}, - } - diffset[kv.StorageDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(3), 0), Value: makeValue(30)}, - } +// --- txNum/epoch unwind invalidation (replaces the blockHash/diffset model) --- - c.RevertWithDiffset(&diffset, blockTip, blockTarget) +// Entries stamped at/below the unwind point survive (warm hot set kept); entries +// above it from the now-dead epoch are dropped lazily on read. +func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + below := makeAddr(1) + above := makeAddr(2) + c.Put(below, makeValue(1), 50) // predates the unwind + c.Put(above, makeValue(2), 150) // written in the unwound range - // Touched keys should be evicted (cache miss). - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "touched account key should be evicted") - _, ok = c.Get(kv.StorageDomain, makeAddr(3)) - assert.False(t, ok, "touched storage key should be evicted") + c.Unwind(100) // keep <=100, drop >100 - // Untouched keys should be preserved. - v, ok := c.Get(kv.AccountsDomain, makeAddr(2)) - assert.True(t, ok, "untouched account key should be preserved") - assert.Equal(t, makeValue(2), v) - v, ok = c.Get(kv.StorageDomain, makeAddr(4)) - assert.True(t, ok, "untouched storage key should be preserved") - assert.Equal(t, makeValue(4), v) + v, ok := c.Get(below) + assert.True(t, ok, "entry below the unwind point must stay warm") + assert.Equal(t, makeValue(1), v) - // Block hash should be updated to the unwind target. - assert.Equal(t, blockTarget, c.GetCache(kv.AccountsDomain).GetBlockHash()) - assert.Equal(t, blockTarget, c.GetCache(kv.StorageDomain).GetBlockHash()) + _, ok = c.Get(above) + assert.False(t, ok, "entry above the unwind point must be invalidated") + assert.Equal(t, 1, c.Len(), "the stale entry is evicted lazily on its read") } -func TestRevertWithDiffset_HashMismatch_ClearsAll(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(3) - blockStale := makeHash(99) // doesn't match any cache - blockTarget := makeHash(1) +// The reused-txNum case: after an unwind, the live fork re-writes a key at the +// SAME txNum as the dead fork's write. The epoch — not the txNum — distinguishes +// them, so the dead entry reads stale and the re-written one reads valid. +func TestUnwind_ReusedTxNumDisambiguatedByEpoch(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + k := makeAddr(1) + c.Put(k, makeValue(1), 150) // dead fork, epoch 0 - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.StorageDomain, makeAddr(2), makeValue(2)) + c.Unwind(100) // epoch -> 1, floor -> 100 - var diffset [kv.DomainLen][]kv.DomainEntryDiff + _, ok := c.Get(k) + assert.False(t, ok, "dead-fork entry (old epoch, above floor) reads stale") - // revertFromHash doesn't match the cache's block hash — full clear. - c.RevertWithDiffset(&diffset, blockStale, blockTarget) - - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "all keys should be cleared on hash mismatch") - _, ok = c.Get(kv.StorageDomain, makeAddr(2)) - assert.False(t, ok, "all keys should be cleared on hash mismatch") - - assert.Equal(t, blockTarget, c.GetCache(kv.AccountsDomain).GetBlockHash()) + c.Put(k, makeValue(2), 150) // live fork re-writes at the same txNum, epoch 1 + v, ok := c.Get(k) + assert.True(t, ok, "live-fork entry at the same txNum is valid (current epoch)") + assert.Equal(t, makeValue(2), v) } -func TestRevertWithDiffset_AccountChange_EvictsCode(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - blockTip := makeHash(5) - blockTarget := makeHash(3) +// A straggler the live fork never re-writes must not resurrect: it stays in a +// dead epoch above the floor and reads stale no matter how far execution +// advances afterwards (there is no rising high-water mark to re-validate it). +func TestUnwind_StragglerNeverResurrects(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + straggler := makeAddr(1) + c.Put(straggler, makeValue(1), 150) // epoch 0 - c.ClearWithHash(blockTip) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.Put(kv.CodeDomain, makeAddr(1), makeCode(1)) - c.Put(kv.CodeDomain, makeAddr(2), makeCode(2)) // untouched + c.Unwind(100) // epoch 1 - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(1), 0), Value: makeValue(10)}, + // Advance the live fork far past the straggler's txNum (no re-write of it). + for i := 2; i < 50; i++ { + c.Put(makeAddr(i), makeValue(i), uint64(200+i)) } - - c.RevertWithDiffset(&diffset, blockTip, blockTarget) - - // Account change evicts both the account and its code. - _, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.False(t, ok, "touched account should be evicted") - _, ok = c.Get(kv.CodeDomain, makeAddr(1)) - assert.False(t, ok, "code for touched account should be evicted") - - // Unrelated code entry preserved. - v, ok := c.Get(kv.CodeDomain, makeAddr(2)) - assert.True(t, ok, "untouched code should be preserved") - assert.Equal(t, makeCode(2), v) + _, ok := c.Get(straggler) + assert.False(t, ok, "straggler in a dead epoch must stay stale, never resurrect") } -func TestRevertWithDiffset_ThenValidateAndPrepare_Continuity(t *testing.T) { - c := NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB) - - block1 := makeHash(1) - block2 := makeHash(2) - block3 := makeHash(3) - block2new := makeHash(22) // new block 2 after re-execution - - // Execute blocks 1, 2, 3. - c.ValidateAndPrepare(common.Hash{}, block1) - c.Put(kv.AccountsDomain, makeAddr(1), makeValue(1)) - c.ValidateAndPrepare(block1, block2) - c.Put(kv.AccountsDomain, makeAddr(2), makeValue(2)) - c.ValidateAndPrepare(block2, block3) - c.Put(kv.AccountsDomain, makeAddr(3), makeValue(3)) - - // Unwind to block 1: revert blocks 2-3. - var diffset [kv.DomainLen][]kv.DomainEntryDiff - diffset[kv.AccountsDomain] = []kv.DomainEntryDiff{ - {Key: makeDiffKey(makeAddr(2), 0), Value: nil}, - {Key: makeDiffKey(makeAddr(3), 0), Value: nil}, - } - c.RevertWithDiffset(&diffset, block3, block1) +// A second, shallower unwind must not resurrect entries a deeper earlier unwind +// invalidated (floor only moves down). +func TestUnwind_FloorOnlyMovesDown(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + k := makeAddr(1) + c.Put(k, makeValue(1), 70) // epoch 0 - // Cache should now have blockHash = block1. - // Executing a new block 2 with parent=block1 should preserve surviving cache data. - valid := c.ValidateAndPrepare(block1, block2new) - assert.True(t, valid, "ValidateAndPrepare should succeed after surgical unwind") + c.Unwind(50) // floor 50, epoch 1 — k(70>50, epoch0) now stale + c.Unwind(100) // shallower; floor must stay 50, not rise to 100 - // addr(1) was not in the diffset — should survive both the revert and ValidateAndPrepare. - v, ok := c.Get(kv.AccountsDomain, makeAddr(1)) - assert.True(t, ok, "untouched key should survive revert + ValidateAndPrepare") - assert.Equal(t, makeValue(1), v) + _, ok := c.Get(k) + assert.False(t, ok, "deeper unwind's floor must not be raised by a later shallower one") } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 163172abda3..fc94bbe04c6 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -17,14 +17,12 @@ package cache import ( - "sync" "sync/atomic" "unsafe" "github.com/c2h5oh/datasize" lru "github.com/hashicorp/golang-lru/v2" - "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" ) @@ -111,9 +109,6 @@ type CodeCache struct { codeSizeEntries atomic.Int64 codeSizeCapEntries int64 - blockHash common.Hash // hash of the last block processed - mu sync.RWMutex - // Stats counters (atomic for concurrent access) addrHits atomic.Uint64 addrMisses atomic.Uint64 @@ -196,7 +191,7 @@ func (c *CodeCache) Get(addr []byte) ([]byte, bool) { // Uses fast maphash to compute the code identifier. // addrToHash is an LRU (auto-evicts oldest when full); hashToCode is byte-capped // and no-ops on new writes when full (code is immutable, so existing entries stay). -func (c *CodeCache) Put(addr []byte, code []byte) { +func (c *CodeCache) Put(addr []byte, code []byte, _ uint64) { if len(code) == 0 { return } @@ -272,7 +267,7 @@ func (c *CodeCache) PutWithEthHash(addr []byte, code []byte, ethHash []byte) { } if len(addr) > 0 { - c.Put(addr, code) + c.Put(addr, code, 0) // addr layer is cleared on unwind, not txNum-tracked } // Populate the size-only layer alongside the bytes layer — every time @@ -334,57 +329,18 @@ func (c *CodeCache) Delete(addr []byte) { // The codeHash → code mappings are preserved since they're immutable. func (c *CodeCache) Clear() { c.addrToHash.Purge() + c.addrToEthHash.Purge() } -// GetBlockHash returns the hash of the last block processed by the cache. -func (c *CodeCache) GetBlockHash() common.Hash { - c.mu.RLock() - defer c.mu.RUnlock() - return c.blockHash -} - -// SetBlockHash sets the hash of the current block being processed. -func (c *CodeCache) SetBlockHash(hash common.Hash) { - c.mu.Lock() - c.blockHash = hash - c.mu.Unlock() -} - -// ValidateAndPrepare checks if the given parentHash matches the cache's current blockHash. -// If there's a mismatch (indicating non-sequential block processing), the address cache is cleared. -// The code cache (hash → code) is preserved since code hashes are immutable. -// Returns true if the cache was valid (hashes matched or cache was empty), false if cleared. -func (c *CodeCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - c.mu.Lock() - defer c.mu.Unlock() - - // Empty blockHash means cache hasn't processed any block yet - always valid - if c.blockHash == (common.Hash{}) { - c.addrToHash.Purge() - c.blockHash = incomingBlockHash - return true - } - - // Check if we're continuing from the expected block - if c.blockHash == parentHash { - c.blockHash = incomingBlockHash - return true - } - - // Mismatch - clear address mappings (they may be stale) - // Keep code mappings since hash → code is immutable - c.addrToHash.Purge() - c.blockHash = incomingBlockHash - return false -} - -// ClearWithHash clears the address cache and sets the block hash. -// Used during unwind to reset the cache to a known state. -func (c *CodeCache) ClearWithHash(hash common.Hash) { - c.mu.Lock() - defer c.mu.Unlock() +// Unwind drops the mutable address → code mappings, which may now reflect a +// dead fork's code for an address. The content-addressed code layers +// (hash → code, ethHash → code, codeSize) are immutable and kept. The addr +// layers are small and clearing them on the rare unwind is cheap, so unlike +// GenericCache they don't need epoch tracking. unwindToTxNum is accepted for +// interface symmetry; the addr layers don't carry per-entry txNums. +func (c *CodeCache) Unwind(_ uint64) { c.addrToHash.Purge() - c.blockHash = hash + c.addrToEthHash.Purge() } // Len returns the number of entries in the address cache. diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index af9e3d61222..0119e855d68 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -18,7 +18,7 @@ package cache import ( "bytes" - "sync" + "math" "sync/atomic" "github.com/c2h5oh/datasize" @@ -43,9 +43,11 @@ const avgBytesPerEntry = 256 // the OnEvict callback can update currentSize without re-running // sizeFunc. type entry[T any] struct { - key []byte - val T - size int + key []byte + val T + size int + txNum uint64 // commit/read txNum the cached value reflects (upper bound) + epoch uint32 // unwind generation the entry was written in } // GenericCache is a sharded, LRU-evicting bounded cache for key-value @@ -56,14 +58,22 @@ type GenericCache[T any] struct { currentSize atomic.Int64 mode Mode - blockHash common.Hash - mu sync.RWMutex - - hits atomic.Uint64 - misses atomic.Uint64 - inserts atomic.Uint64 - evictions atomic.Uint64 - dropped atomic.Uint64 + // Cache coherence across unwinds is txNum/epoch based — no block awareness, + // no diffset. An entry is valid iff it was written in the current epoch OR + // its txNum is at/below unwindFloor (so it predates every unwind seen and + // can't be stale). Unwind bumps the epoch and lowers the floor (O(1), no + // scan); stale entries are dropped lazily on their next read. txNum slots + // are reused across forks, so the epoch — not the txNum — is what tells a + // dead fork's write apart from the live fork's at the same txNum. + epoch atomic.Uint32 + unwindFloor atomic.Uint64 + + hits atomic.Uint64 + misses atomic.Uint64 + inserts atomic.Uint64 + evictions atomic.Uint64 + dropped atomic.Uint64 + staleEvicted atomic.Uint64 // entries dropped lazily on read after an unwind sizeFunc func(T) int } @@ -101,6 +111,9 @@ func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntr mode: mode, sizeFunc: sizeFunc, } + // Before any unwind every entry predates the (nonexistent) floor, so all + // reads are valid; the floor only drops once an unwind happens. + c.unwindFloor.Store(math.MaxUint64) // OnEvict fires for capacity-driven LRU eviction (ModeEvictLRU) and // for explicit Remove(). In both cases we want currentSize to follow. lru.SetOnEvict(func(_ uint64, e entry[T]) { @@ -138,8 +151,8 @@ func (c *DomainCache) Get(key []byte) ([]byte, bool) { } // Put stores data for the given key, implementing the Cache interface. -func (c *DomainCache) Put(key []byte, value []byte) { - c.GenericCache.Put(key, value) +func (c *DomainCache) Put(key []byte, value []byte, txNum uint64) { + c.GenericCache.Put(key, value, txNum) } // Delete removes the data for the given key, delegating to GenericCache. @@ -156,6 +169,16 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { var zero T return zero, false } + // Lazy unwind invalidation: an entry from a superseded epoch whose txNum is + // above the unwind floor reflects dead-fork state — drop it and miss so the + // read falls through to the reverted domain and repopulates. + if e.epoch != c.epoch.Load() && e.txNum > c.unwindFloor.Load() { + c.data.Remove(h) + c.staleEvicted.Add(1) + c.misses.Add(1) + var zero T + return zero, false + } c.hits.Add(1) return e.val, true } @@ -164,17 +187,18 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { // sharded LRU evicts cold entries when its entry-count cap is reached. // In ModeNoOp inserts that would overflow the byte budget are dropped // (and counted via the dropped metric). -func (c *GenericCache[T]) Put(key []byte, value T) { +func (c *GenericCache[T]) Put(key []byte, value T, txNum uint64) { h := maphash.Hash(key) valBytes := c.sizeFunc(value) newSize := len(key) + valBytes + 24 + ep := c.epoch.Load() // Existing key — update in place. Reuse the stored key buffer to // avoid an extra allocation; the freshly-decoded value replaces the // old one. if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { oldSize := existing.size - c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize}) + c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize - oldSize)) return } @@ -191,7 +215,7 @@ func (c *GenericCache[T]) Put(key []byte, value T) { // trade-off code_cache.go / balcache.go / db/state/cache.go accept. keyCopy := common.Copy(key) - c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize}) + c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize)) c.inserts.Add(1) } @@ -210,50 +234,24 @@ func (c *GenericCache[T]) Clear() { c.currentSize.Store(0) } -// GetBlockHash returns the hash of the last block processed by the cache. -func (c *GenericCache[T]) GetBlockHash() common.Hash { - c.mu.RLock() - defer c.mu.RUnlock() - return c.blockHash -} - -// SetBlockHash sets the hash of the current block being processed. -func (c *GenericCache[T]) SetBlockHash(hash common.Hash) { - c.mu.Lock() - c.blockHash = hash - c.mu.Unlock() -} - -// ValidateAndPrepare checks if the given parentHash matches the cache's current blockHash. -func (c *GenericCache[T]) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - c.mu.Lock() - defer c.mu.Unlock() - - if c.blockHash == (common.Hash{}) { - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = incomingBlockHash - return true - } - - if c.blockHash == parentHash { - c.blockHash = incomingBlockHash - return true +// Unwind invalidates entries that reflect state above unwindToTxNum on a +// now-dead fork. O(1): bump the epoch (so entries written in the new, live +// epoch stay valid) and lower the floor to the unwind point (so entries at or +// below it — which predate the unwind and can't be stale — stay warm). Stale +// entries (old epoch, txNum above the floor) are dropped lazily on their next +// read. The floor only ever moves down, to the deepest unwind seen, so a later +// shallower unwind can't resurrect entries an earlier deeper one invalidated. +func (c *GenericCache[T]) Unwind(unwindToTxNum uint64) { + c.epoch.Add(1) + for { + cur := c.unwindFloor.Load() + if unwindToTxNum >= cur { + break + } + if c.unwindFloor.CompareAndSwap(cur, unwindToTxNum) { + break + } } - - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = incomingBlockHash - return false -} - -// ClearWithHash clears the cache and sets the block hash. -func (c *GenericCache[T]) ClearWithHash(hash common.Hash) { - c.mu.Lock() - defer c.mu.Unlock() - c.data.Purge() - c.currentSize.Store(0) - c.blockHash = hash } // Len returns the number of entries in the cache. @@ -283,6 +281,7 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { inserts := c.inserts.Swap(0) evictions := c.evictions.Swap(0) dropped := c.dropped.Swap(0) + staleEvicted := c.staleEvicted.Swap(0) total := hits + misses var hitRate float64 if total > 0 { @@ -290,10 +289,11 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { } sizeBytes := c.currentSize.Load() usagePct := float64(sizeBytes) / float64(c.capacityB) * 100 - log.Debug(name+" cache stats", + log.Info(name+" cache stats", "mode", c.mode.String(), "hits", hits, "misses", misses, "hit_rate", hitRate, "inserts", inserts, "evictions", evictions, "dropped", dropped, + "stale_evicted", staleEvicted, "epoch", c.epoch.Load(), "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", c.capacityB/datasize.MB, "usage_pct", usagePct, ) diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index dbf72fb3114..da5a6a53fe7 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -32,8 +32,11 @@ import ( const ( // DefaultAccountCacheBytes is the byte limit for account cache (100 MB — investigation knob; permanent default returns to 1 GB) DefaultAccountCacheBytes = 100 * datasize.MB - // DefaultStorageCacheBytes is the byte limit for storage cache (100 MB — investigation knob; permanent default returns to 1 GB) - DefaultStorageCacheBytes = 100 * datasize.MB + // DefaultStorageCacheBytes is the byte limit for storage cache. 150 MB: the + // measured mainnet-tip storage working set is ~106 MB for 95% of reads + // (top 1% of keys = 48%, ~10 MB for 80%); 150 MB leaves headroom over the + // 95% set so eviction pressure doesn't push the hot set out. + DefaultStorageCacheBytes = 150 * datasize.MB // DefaultCommitmentCacheBytes is the byte limit for commitment cache (128 MB) DefaultCommitmentCacheBytes = 128 * datasize.MB @@ -204,8 +207,9 @@ func (c *StateCache) DeleteAddrCodeHash(addr []byte) { cc.DeleteAddrCodeHash(addr) } -// Put stores data for the given domain and key. -func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { +// Put stores data for the given domain and key, stamped with the txNum the +// value reflects (for txNum/epoch unwind invalidation). +func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte, txNum uint64) { cache := c.caches[domain] if cache == nil { return @@ -213,7 +217,7 @@ func (c *StateCache) Put(domain kv.Domain, key []byte, value []byte) { if domain == kv.CommitmentDomain && bytes.Equal(key, commitmentdb.KeyCommitmentState) { return } - cache.Put(key, common.Copy(value)) + cache.Put(key, common.Copy(value), txNum) } // Delete removes the data for the given domain and key. @@ -234,25 +238,15 @@ func (c *StateCache) Clear() { } } -// ValidateAndPrepare validates and prepares all caches for a new block. -// Returns true if all caches were valid, false if any were cleared. -func (c *StateCache) ValidateAndPrepare(parentHash common.Hash, incomingBlockHash common.Hash) bool { - allValid := true +// Unwind invalidates, across all caches, entries reflecting state above +// unwindToTxNum on a now-dead fork. Diffset-free and O(1) (GenericCache bumps +// an epoch + lowers a floor; CodeCache clears its mutable addr layers). This is +// the sole cache-invalidation path on unwind — the executor never touches the +// cache during forward execution. +func (c *StateCache) Unwind(unwindToTxNum uint64) { for _, cache := range c.caches { if cache != nil { - if !cache.ValidateAndPrepare(parentHash, incomingBlockHash) { - allValid = false - } - } - } - return allValid -} - -// ClearWithHash clears all caches and sets their block hash. -func (c *StateCache) ClearWithHash(hash common.Hash) { - for _, cache := range c.caches { - if cache != nil { - cache.ClearWithHash(hash) + cache.Unwind(unwindToTxNum) } } } @@ -284,44 +278,3 @@ func (c *StateCache) PrintStatsAndReset() { code.PrintStatsAndReset() } } - -func (c *StateCache) RevertWithDiffset(diffset *[6][]kv.DomainEntryDiff, revertFromHash, newBlockHash common.Hash) { - // If the cache's block hash doesn't match the block we're unwinding from, - // the cache was modified by a rolled-back tx (e.g. ValidatePayload). - // Clear everything and set the new hash — surgical eviction can't fix stale data - // from a different execution path. - for _, cache := range c.caches { - if cache != nil && cache.GetBlockHash() != revertFromHash { - c.ClearWithHash(newBlockHash) - return - } - } - - for _, entry := range diffset[kv.AccountsDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CodeDomain, k) - c.Delete(kv.AccountsDomain, k) - // Unwind may revert a codeHash change (SELFDESTRUCT undone, CREATE - // reverted) — drop the addr → codeHash mapping so the next read - // repopulates from the canonical post-revert account record. - c.DeleteAddrCodeHash(k) - } - for _, entry := range diffset[kv.CodeDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CodeDomain, k) - } - for _, entry := range diffset[kv.StorageDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.StorageDomain, k) - } - for _, entry := range diffset[kv.CommitmentDomain] { - k := []byte(entry.Key[:len(entry.Key)-8]) - c.Delete(kv.CommitmentDomain, k) - } - // Update block hash after successful surgical eviction - for _, cache := range c.caches { - if cache != nil { - cache.SetBlockHash(newBlockHash) - } - } -} diff --git a/execution/exec/blocks_read_ahead.go b/execution/exec/blocks_read_ahead.go index 08c0d47a5c9..57931161f85 100644 --- a/execution/exec/blocks_read_ahead.go +++ b/execution/exec/blocks_read_ahead.go @@ -85,6 +85,7 @@ func (bra *BlockReadAheader) SetStateCache(sc *cache.StateCache) { type cachePopulatingGetter struct { g kv.TemporalGetter sc *cache.StateCache + stepSize uint64 // for the read txNum upper bound (last txNum of the read's step) codeHashHint []byte // valid only for the next CodeDomain read; cleared after use } @@ -100,7 +101,9 @@ func (cpg *cachePopulatingGetter) GetLatest(name kv.Domain, k []byte) ([]byte, k // storage slot, no code) and caching it lets repeated probes // skip the file accessor stack. Mirrors revm's CacheAccount // { account: None, status: LoadedNotExisting } pattern. - cpg.sc.Put(name, k, v) + // Stamp with an upper bound on the value's write txNum (last txNum + // of the step it came from) so unwind invalidation is correct. + cpg.sc.Put(name, k, v, (uint64(step)+1)*cpg.stepSize-1) } } return v, step, err @@ -229,7 +232,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} getter = cpg } stateReader := state.NewReaderV3(getter) @@ -308,7 +311,7 @@ func (bra *BlockReadAheader) warmBody(ctx context.Context, db kv.RoDB, header *t var getter kv.TemporalGetter = ttx var cpg *cachePopulatingGetter if bra.stateCache != nil { - cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache} + cpg = &cachePopulatingGetter{g: ttx, sc: bra.stateCache, stepSize: ttx.Debug().StepSize()} getter = cpg } stateReader := state.NewReaderV3(getter) diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 8a46de7e839..f20e02a2125 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -555,11 +555,11 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b return ValidationResult{}, criticalError } - // Clear state cache on invalid block - isInvalid := status == engine_types.InvalidStatus || status == engine_types.InvalidBlockHashStatus || validationError != nil - if e.stateCache != nil && isInvalid { - e.stateCache.ClearWithHash(header.ParentHash) - } + // No cache invalidation needed on an invalid payload: the state cache is + // populated only at flush (committed, fork-agnostic state) and this + // validation path never flushes, so a rejected payload leaves nothing + // fork-specific in the cache. Reads during validation only add canonical + // committed bytes. (Cache invalidation happens solely on unwind.) // Validation tx is the SD's BlockOverlay; defer doms.Close() above handles // its rollback. By design we do not persist validation-run writes — there diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5d2e668794c..e57af3c0c53 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -600,13 +600,6 @@ func (te *txExecutor) executeBlocks(ctx context.Context, startBlockNum uint64, m } go warmTxsHashes(b) - // Bug 2 fix from commit 43a64a0b66 (parallel executor: fix two cache bugs): - // validate stateCache entries against this block's parent hash so stale - // values from a prior fork-validation are cleared before reads. - if stateCache := te.doms.GetStateCache(); stateCache != nil { - stateCache.ValidateAndPrepare(b.ParentHash(), b.Hash()) - } - var dbBAL types.BlockAccessList // Read BAL through blockTx (overlay or execRoTx) — do NOT open // a separate db.View() as it can deadlock with the stageloop's diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 0b5a18f89b8..1eca858a1f2 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -728,6 +728,9 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U func (pe *parallelExecutor) LogExecution() { pe.progress.LogExecution(pe.rs.StateV3, pe) + if stateCache := pe.doms.GetStateCache(); stateCache != nil { + stateCache.PrintStatsAndReset() + } if domainMetrics := pe.domains().LogMetrics(); len(domainMetrics) > 0 { pe.logger.Info(fmt.Sprintf("[%s] domain reads", pe.logPrefix), domainMetrics...) } @@ -1055,17 +1058,10 @@ func (pe *parallelExecutor) execLoop(ctx context.Context) (err error) { } func (pe *parallelExecutor) processRequest(ctx context.Context, execRequest *execRequest) (err error) { - // Validate state cache before processing block - ensures cache is cleared after reorgs - // This matches the behavior in serial execution (exec3_serial.go) - if len(execRequest.tasks) > 0 { - if txTask, ok := execRequest.tasks[0].(*exec.TxTask); ok && txTask.Header != nil { - parentHash := txTask.Header.ParentHash - blockHash := execRequest.blockHash - if stateCache := pe.doms.GetStateCache(); stateCache != nil { - stateCache.ValidateAndPrepare(parentHash, blockHash) - } - } - } + // The state cache is a SharedDomain implementation detail: it is populated + // only at flush (committed, fork-agnostic state) and invalidated only on + // unwind (txNum/epoch — see StateCache.Unwind). The executor does not touch + // it during forward execution. prevSenderTx := map[accounts.Address]int{} var scheduleable *blockExecutor diff --git a/execution/stagedsync/exec3_serial.go b/execution/stagedsync/exec3_serial.go index a2bff285df0..7c53ad1b82a 100644 --- a/execution/stagedsync/exec3_serial.go +++ b/execution/stagedsync/exec3_serial.go @@ -82,7 +82,7 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw "initialCycle", initialCycle, "isForkValidation", se.isForkValidation) } - stateCache := se.doms.GetStateCache() + stateCache := se.doms.GetStateCache() // for periodic PrintStatsAndReset only for ; blockNum <= maxBlockNum; blockNum++ { shouldGenerateChangesets := shouldGenerateChangeSets(se.cfg, blockNum, maxBlockNum) @@ -114,10 +114,6 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw } go warmTxsHashes(b) - if stateCache != nil { - stateCache.ValidateAndPrepare(b.ParentHash(), b.Hash()) - } - txs := b.Transactions() header := b.HeaderNoCopy() getHashFnMutex := sync.Mutex{} diff --git a/execution/stagedsync/stage_execute.go b/execution/stagedsync/stage_execute.go index dccf76087fa..2b01df83d3e 100644 --- a/execution/stagedsync/stage_execute.go +++ b/execution/stagedsync/stage_execute.go @@ -212,8 +212,7 @@ func unwindExec3(u *UnwindState, s *StageState, doms *execctx.SharedDomains, rwT } } } - // Get the hash of the last executed block (the tip we're unwinding from) - // so RevertWithDiffset can detect if the cache was modified by a rolled-back tx. + // Get the hash of the last executed block (the tip we're unwinding from). lastExecHash, _, err := br.CanonicalHash(ctx, rwTx, u.CurrentBlockNumber) if err != nil { lastExecHash = common.Hash{} @@ -221,14 +220,12 @@ func unwindExec3(u *UnwindState, s *StageState, doms *execctx.SharedDomains, rwT if err := unwindExec3State(ctx, doms, rwTx, u.UnwindPoint, txNum, accumulator, changeSet, lastExecHash, logger); err != nil { return fmt.Errorf("unwindExec3State(%d->%d): %w, took %s", s.BlockNumber, u.UnwindPoint, err, time.Since(t)) } - // Surgically evict keys touched by the unwound blocks from the state cache. - // Keys not in the diffset remain cached (they weren't modified by the unwound range). - if stateCache := doms.GetStateCache(); stateCache != nil && changeSet != nil { - unwindTargetHash, _, err := br.CanonicalHash(ctx, rwTx, u.UnwindPoint) - if err != nil { - unwindTargetHash = common.Hash{} - } - stateCache.RevertWithDiffset(changeSet, lastExecHash, unwindTargetHash) + // Invalidate the state cache for everything above the unwind point. txNum/epoch + // based and diffset-free (see StateCache.Unwind), so it runs unconditionally — + // independent of whether changesets were generated for the unwound range, which + // they are not below the reorg window. Matches the domain overlay's maxtx prune. + if stateCache := doms.GetStateCache(); stateCache != nil { + stateCache.Unwind(txNum) } if err := rawdb.DeleteNewerEpochs(rwTx, u.UnwindPoint+1); err != nil { return fmt.Errorf("delete newer epochs: %w", err) diff --git a/execution/vm/contract.go b/execution/vm/contract.go index da7e2cb9c3b..b80bb53e912 100644 --- a/execution/vm/contract.go +++ b/execution/vm/contract.go @@ -105,7 +105,8 @@ func (c *Contract) isCode(udest uint64) bool { c.analysis = codeBitmap(c.Code) if !isCodeHashZero { - jumpDestCache.Put(codeHash[:], c.analysis) + // content-addressed by codeHash and never unwound, so txNum is irrelevant + jumpDestCache.Put(codeHash[:], c.analysis, 0) } return c.analysis.codeSegment(udest) From 42a2aafd32458127e2f16f1adece8f041225ded7 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 18:01:58 +0000 Subject: [PATCH 112/120] execution: wire state cache into SetHead unwind (fixes BadBlock on re-exec) SetHead (debug_setHead) built its unwind SharedDomains via NewSharedDomains but never called SetStateCache, so unwindExec3's doms.GetStateCache() returned nil and stateCache.Unwind(txNum) was skipped. The shared singleton cache kept pre-unwind values at the old epoch; the next UpdateForkChoice re-execution read them and computed a stale state root, returning ExecutionStatusBadBlock. Wire the shared cache into the SetHead SD, mirroring ValidateChain and the forkchoice path, so the epoch bump + floor lowering runs. TestSetHeadCanonicalCleanup covers this (failed cache-on, passes now). Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/execmodule/set_head.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 0623b5bcaa6..75aef16ff7a 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -103,6 +103,12 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { } defer sd.Close() + // Wire the shared state cache so unwindExec3 invalidates it (epoch bump + + // floor lower). Without this, GetStateCache() returns nil during the unwind, + // the cache keeps pre-unwind values, and the next FCU re-execution reads them + // and computes a stale state root (BadBlock). Mirrors ValidateChain/forkchoice. + sd.SetStateCache(e.stateCache) + // Set the unwind point and run the unwind if err := e.pipelineExecutor.UnwindTo(targetBlock, stagedsync.StagedUnwind, tx); err != nil { return fmt.Errorf("failed to set unwind point: %w", err) From 26838656780069d46aecb73a6d7ed8b690c28dd5 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Fri, 5 Jun 2026 18:38:31 +0000 Subject: [PATCH 113/120] commitment: LatestStateReader.Clone keeps its source tx, not the compute tx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ComputeCommitment clones a custom state reader with its own compute tx (commitment_context.go trieContext). When that reader is a LatestStateReader bound to a different database (e.g. recomputing commitment in an empty db while reading committed state from the source db, as TouchChangedKeysFromHistory does), Clone rebuilt sd.AsGetter against the foreign compute tx and read the wrong database — returning empty account/storage and thus a wrong (empty-leaf) root. This was masked until FlushWithCallback began draining sd.mem on flush: the in-memory batch previously still held the source values. Store the reader's source tx and rebind Clone to it. The default commitment path never routes through LatestStateReader.Clone (it constructs fresh readers), and the production custom readers build their own sub-readers, so this only affects readers explicitly bound to a foreign source. Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/commitment/commitmentdb/reader.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/execution/commitment/commitmentdb/reader.go b/execution/commitment/commitmentdb/reader.go index d2950323f53..805931727db 100644 --- a/execution/commitment/commitmentdb/reader.go +++ b/execution/commitment/commitmentdb/reader.go @@ -16,10 +16,11 @@ type StateReader interface { type LatestStateReader struct { sharedDomains sd getter kv.TemporalGetter + srcTx kv.TemporalTx } func NewLatestStateReader(tx kv.TemporalTx, sd sd) *LatestStateReader { - return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx)} + return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx} } func (r *LatestStateReader) WithHistory() bool { @@ -43,7 +44,14 @@ func (r *LatestStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) } func (r *LatestStateReader) Clone(tx kv.TemporalTx) StateReader { - return NewLatestStateReader(tx, r.sharedDomains) + // Keep reading the source this reader was bound to. The tx passed by + // clone/warmup callers targets the *compute* database, which may differ + // from this reader's source — e.g. recomputing commitment in an empty db + // while reading committed state from the source db (TouchChangedKeysFromHistory). + // Before flush drained sd.mem this was masked because the in-memory batch + // still held the source values; rebinding sd.AsGetter to the foreign compute + // tx reads the wrong database and yields empty state (wrong root). + return NewLatestStateReader(r.srcTx, r.sharedDomains) } // HistoryStateReader reads *full* historical state at specified txNum. From 44355fd1fba362c60da48a087142ad51e168d797 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 19:45:10 +0000 Subject: [PATCH 114/120] db/state, commitment: lock-free per-worker read metrics (+ tx-aware BranchCache) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the DomainMetrics write lock from the hot state-read path. Every GetLatest previously took DomainMetrics.Lock() — a write lock with no fast path — to bump read counters; under concurrent trie-warmup / parallel-exec workers that was a hard serialization point, so KV read metrics had to be left off. Replace it with a lock-free per-worker accumulator (changeset.WorkerMetrics) that each goroutine owns exclusively, combined into the shared DomainMetrics once per task via DomainMetrics.Merge — no lock, no atomics, no shared cache line on the hot path. The accumulator is carried via context (const-int ctx key, matching node/app) and read through getter.GetLatestContext / SharedDomains.GetLatestContext (mirroring the existing optional MeteredGetLatest). Commitment/warmup worker readers carry a per-worker ctx and merge at teardown; the main goroutine meters into sd.mainWM, flushed into the global on metrics read. Parallel-exec state readers use AsGetterNoMetrics for now (no metering); wiring their per-worker metrics is a follow-up on the state-read path. This also removes a data race those concurrent workers would otherwise hit on a shared accumulator once metrics are enabled. Also carries the tx-aware BranchCache (txNum/epoch unwind invalidation + maxStep-bounded reads) this sits on top of. Verified: build, make lint (0 issues), and -race with KV_READ_METRICS=true clean across commitmentdb / changeset / execmodule parallel block-assembly. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/aggregator.go | 4 +- db/state/changeset/state_changeset.go | 282 ++++++++++++------ db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 159 ++++++++-- execution/commitment/branch_cache.go | 94 +++++- .../commitmentdb/commitment_context.go | 26 +- .../commitmentdb/commitment_context_test.go | 3 + execution/commitment/commitmentdb/reader.go | 85 +++++- execution/commitment/preload.go | 6 +- execution/exec/state.go | 10 +- execution/execmodule/forkchoice.go | 2 + execution/stagedsync/committer.go | 19 +- execution/stagedsync/exec3_parallel.go | 4 +- rpc/jsonrpc/eth_simulation.go | 7 + 14 files changed, 555 insertions(+), 148 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 387e4f4a906..c83c8568dbd 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2478,11 +2478,11 @@ func (at *AggregatorRoTx) GetLatest(domain kv.Domain, k []byte, tx kv.Tx) (v []b return at.getLatest(domain, k, tx, math.MaxUint64, nil, time.Time{}) } -func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { return at.getLatest(domain, k, tx, maxStep, metrics, start) } -func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { if domain != kv.CommitmentDomain { return at.d[domain].getLatest(k, tx, maxStep, metrics, start) } diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 0e30f687897..60bbd70a3d4 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -17,6 +17,7 @@ package changeset import ( + "context" "encoding/binary" "fmt" "math" @@ -514,79 +515,189 @@ type DomainMetrics struct { seenFileReads sync.Map } -func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.CacheReadCount++ - readDuration := time.Since(start) - dm.CacheReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.CacheReadCount++ - d.CacheReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - CacheReadCount: 1, - CacheReadDuration: readDuration, - } +// WorkerMetrics is a per-worker, lock-free I/O metrics accumulator. Each +// goroutine on the hot read path (the main commitment/exec goroutine and each +// trie-warmup worker) owns one exclusively, so updates need no synchronization +// — no mutex, no atomics, no shared cache line. The per-worker accumulators are +// combined into the shared DomainMetrics once per task via DomainMetrics.Merge. +// +// This replaces the previous design where every read called a DomainMetrics +// method that took the global write lock (dm.Lock()); under concurrent warmup +// workers that write lock was a hard serialization point on the hottest path. +type WorkerMetrics struct { + total DomainIOMetrics + domains [kv.DomainLen]DomainIOMetrics + + // uniqueSeen records the first local sighting of each (domain,prefix) that + // fell through to a file read, so cross-worker uniqueness can be resolved + // once at Merge time (against DomainMetrics.seenFileReads) instead of + // touching a shared structure on every read. Value is the length bucket. + // Nil until the first UpdateFileReadsUnique call (workers that never read + // from files pay nothing). + uniqueSeen map[uniqueKey]int +} + +type uniqueKey struct { + domain kv.Domain + prefix string +} + +// NewWorkerMetrics returns a fresh lock-free accumulator. A nil *WorkerMetrics +// is valid and makes every Update* a no-op, so callers that don't collect +// metrics can pass nil. +func NewWorkerMetrics() *WorkerMetrics { return &WorkerMetrics{} } + +// ctxKey is the typed-int context-key namespace for this package (the const-int +// model used by node/app), avoiding per-key struct types. +type ctxKey int + +const ( + ckWorkerMetrics ctxKey = iota +) + +// ContextWithWorkerMetrics returns a child context carrying a per-worker, +// lock-free metrics accumulator. Concurrent workers (e.g. trie-warmup +// goroutines) each run with their own, so reads never contend on a shared +// metrics structure (and take no lock). +func ContextWithWorkerMetrics(ctx context.Context, wm *WorkerMetrics) context.Context { + return context.WithValue(ctx, ckWorkerMetrics, wm) +} + +// WorkerMetricsFromContext extracts the per-worker accumulator, or nil if none +// was attached (in which case reads collect no metrics — a valid no-op). +func WorkerMetricsFromContext(ctx context.Context) *WorkerMetrics { + if ctx == nil { + return nil } + wm, _ := ctx.Value(ckWorkerMetrics).(*WorkerMetrics) + return wm } -func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.DbReadCount++ - readDuration := time.Since(start) - dm.DbReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.DbReadCount++ - d.DbReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - DbReadCount: 1, - DbReadDuration: readDuration, - } +// Reset zeroes the accumulator so it can be reused after its counts have been +// merged into the shared DomainMetrics. Used for the long-lived main-goroutine +// accumulator, which is merged-and-reset on each metrics read. +func (w *WorkerMetrics) Reset() { + if w == nil { + return } + w.total = DomainIOMetrics{} + for i := range w.domains { + w.domains[i] = DomainIOMetrics{} + } + w.uniqueSeen = nil } -func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { - dm.Lock() - defer dm.Unlock() - dm.StateCacheHitCount++ - if d, ok := dm.Domains[domain]; ok { - d.StateCacheHitCount++ - } else { - dm.Domains[domain] = &DomainIOMetrics{StateCacheHitCount: 1} +func (w *WorkerMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { + if w == nil { + return } + d := time.Since(start) + w.total.CacheReadCount++ + w.total.CacheReadDuration += d + w.domains[domain].CacheReadCount++ + w.domains[domain].CacheReadDuration += d } -func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { - dm.Lock() - defer dm.Unlock() - dm.StateCacheMissCount++ - if d, ok := dm.Domains[domain]; ok { - d.StateCacheMissCount++ - } else { - dm.Domains[domain] = &DomainIOMetrics{StateCacheMissCount: 1} +func (w *WorkerMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.DbReadCount++ + w.total.DbReadDuration += d + w.domains[domain].DbReadCount++ + w.domains[domain].DbReadDuration += d +} + +func (w *WorkerMetrics) UpdateStateCacheHit(domain kv.Domain) { + if w == nil { + return } + w.total.StateCacheHitCount++ + w.domains[domain].StateCacheHitCount++ } -func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { +func (w *WorkerMetrics) UpdateStateCacheMiss(domain kv.Domain) { + if w == nil { + return + } + w.total.StateCacheMissCount++ + w.domains[domain].StateCacheMissCount++ +} + +func (w *WorkerMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.FileReadCount++ + w.total.FileReadDuration += d + w.domains[domain].FileReadCount++ + w.domains[domain].FileReadDuration += d +} + +// Merge folds a worker's accumulated counters into the shared DomainMetrics +// under a single lock acquisition (once per task, not once per read). Unique +// file-read prefixes are deduplicated across workers here, against the global +// seenFileReads set, so the cross-worker uniqueness count stays correct without +// any shared structure on the hot path. +func (dm *DomainMetrics) Merge(w *WorkerMetrics) { + if w == nil { + return + } dm.Lock() defer dm.Unlock() - dm.FileReadCount++ - readDuration := time.Since(start) - dm.FileReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.FileReadCount++ - d.FileReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - FileReadCount: 1, - FileReadDuration: readDuration, + addIOMetrics(&dm.DomainIOMetrics, &w.total) + for d := range w.domains { + src := &w.domains[d] + if src.isZero() { + continue + } + dst, ok := dm.Domains[kv.Domain(d)] + if !ok { + dst = &DomainIOMetrics{} + dm.Domains[kv.Domain(d)] = dst + } + addIOMetrics(dst, src) + } + for uk, bucket := range w.uniqueSeen { + domainKey := uk.domain.String() + ":" + uk.prefix + if _, already := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}); already { + continue + } + dm.UniqueFileReadCount++ + dm.UniqueLenBuckets[bucket]++ + if dst, ok := dm.Domains[uk.domain]; ok { + dst.UniqueFileReadCount++ + dst.UniqueLenBuckets[bucket]++ + } else { + nd := &DomainIOMetrics{UniqueFileReadCount: 1} + nd.UniqueLenBuckets[bucket] = 1 + dm.Domains[uk.domain] = nd } } } +// addIOMetrics adds src's cumulative read-side counters into dst. CachePut* +// fields are intentionally excluded: those track mem-batch buffer occupancy +// (functional state, reset on flush), not cumulative read metrics, and are +// accounted separately off this path. +func addIOMetrics(dst, src *DomainIOMetrics) { + dst.CacheReadCount += src.CacheReadCount + dst.CacheReadDuration += src.CacheReadDuration + dst.DbReadCount += src.DbReadCount + dst.DbReadDuration += src.DbReadDuration + dst.FileReadCount += src.FileReadCount + dst.FileReadDuration += src.FileReadDuration + dst.StateCacheHitCount += src.StateCacheHitCount + dst.StateCacheMissCount += src.StateCacheMissCount +} + +func (m *DomainIOMetrics) isZero() bool { + return m.CacheReadCount == 0 && m.DbReadCount == 0 && m.FileReadCount == 0 && + m.StateCacheHitCount == 0 && m.StateCacheMissCount == 0 +} + // lenBucket maps a prefix byte-length to its UniqueLenBuckets index. // See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets // 5-8 sub-divide the storage subtree (depths 64-127) so per-contract @@ -616,45 +727,26 @@ func lenBucket(n int) int { } } -// UpdateFileReadsUnique records a file read while also tracking whether -// the prefix has been seen before for this domain. Same gate as -// UpdateFileReads (dbg.KVReadLevelledMetrics); call this instead when -// the read-amplification ratio (FileReadCount / UniqueFileReadCount) -// is wanted on the metric output. The key bytes are copied into the -// internal dedup map; do not mutate after the call. -func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { - // Composite ":" so two domains can hold the same prefix - // shape (commitment compact-encoded paths vs accounts plain addresses) - // without colliding. - domainKey := domain.String() + ":" + string(key) - _, alreadySeen := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}) - bucket := lenBucket(len(key)) - - dm.Lock() - defer dm.Unlock() - dm.FileReadCount++ - readDuration := time.Since(start) - dm.FileReadDuration += readDuration - if !alreadySeen { - dm.UniqueFileReadCount++ - dm.UniqueLenBuckets[bucket]++ - } - if d, ok := dm.Domains[domain]; ok { - d.FileReadCount++ - d.FileReadDuration += readDuration - if !alreadySeen { - d.UniqueFileReadCount++ - d.UniqueLenBuckets[bucket]++ - } - } else { - newD := &DomainIOMetrics{ - FileReadCount: 1, - FileReadDuration: readDuration, - } - if !alreadySeen { - newD.UniqueFileReadCount = 1 - newD.UniqueLenBuckets[bucket] = 1 - } - dm.Domains[domain] = newD +// UpdateFileReadsUnique records a file read in the worker-local accumulator, +// remembering the (domain,prefix) first-sighting so cross-worker uniqueness can +// be resolved at Merge time. Lock-free: the key bytes are copied into a +// per-worker map; nothing shared is touched on the hot path. The key bytes are +// copied; do not mutate after the call. +func (w *WorkerMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + if w == nil { + return + } + d := time.Since(start) + w.total.FileReadCount++ + w.total.FileReadDuration += d + w.domains[domain].FileReadCount++ + w.domains[domain].FileReadDuration += d + + if w.uniqueSeen == nil { + w.uniqueSeen = make(map[uniqueKey]int) + } + uk := uniqueKey{domain: domain, prefix: string(key)} + if _, ok := w.uniqueSeen[uk]; !ok { + w.uniqueSeen[uk] = lenBucket(len(key)) } } diff --git a/db/state/domain.go b/db/state/domain.go index d9158f5ce42..e512287659e 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1611,7 +1611,7 @@ func (dt *DomainRoTx) GetLatest(key []byte, roTx kv.Tx) ([]byte, kv.Step, bool, return dt.getLatest(key, roTx, math.MaxInt64, nil, time.Time{}) } -func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) ([]byte, kv.Step, bool, error) { +func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) ([]byte, kv.Step, bool, error) { if dt.d.Disable { return nil, 0, false, nil } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index a184e56219c..073ecf3b0d9 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "math" + "os" "runtime" "strings" "sync" @@ -108,6 +109,11 @@ type SharedDomains struct { disableInlineTouchKey bool mem kv.TemporalMemBatch metrics changeset.DomainMetrics + // mainWM is the lock-free metrics accumulator for the main goroutine's + // reads (direct GetLatest + the main commitment/exec getter). Trie-warmup + // workers get their own via AsGetterWorker; all are combined into metrics + // at task boundaries. Keeps the global metrics write-lock off the hot path. + mainWM *changeset.WorkerMetrics // blockOverlay is an in-memory overlay for block-level metadata writes (headers, bodies, // canonical hashes, TD, stage progress, forkchoice markers). It allows execution to @@ -197,6 +203,7 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg logger: logger, //trace: true, metrics: changeset.DomainMetrics{Domains: map[kv.Domain]*changeset.DomainIOMetrics{}}, + mainWM: changeset.NewWorkerMetrics(), stepSize: tx.Debug().StepSize(), } @@ -433,10 +440,26 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx + // wm is the accumulator GetLatest records into: sd.mainWM for the main + // goroutine (AsGetter), or nil to collect nothing (AsGetterNoMetrics) for + // concurrent callers whose per-worker metering isn't wired yet (parallel + // exec workers) — they must not write the shared main accumulator (a race). + wm *changeset.WorkerMetrics } func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.GetLatest(name, gt.tx, k) + return gt.sd.getLatestMetered(name, gt.tx, k, gt.wm) +} + +// GetLatestContext is the context-aware read: it records into the per-worker, +// lock-free accumulator carried by ctx (a nil ctx-value collects no metrics). +// Concurrent workers (trie-warmup goroutines) pass their own accumulator via +// ctx, so they neither share metrics state with the main goroutine nor take any +// lock. Optional method — callers type-assert for it (mirrors the existing +// AggregatorRoTx.MeteredGetLatest pattern). The plan is to migrate GetLatest +// callers onto this ctx-carrying form in a follow-up, then drop the split. +func (gt *temporalGetter) GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { + return gt.sd.getLatestMetered(name, gt.tx, k, changeset.WorkerMetricsFromContext(ctx)) } // GetCodeSize returns the length of the code at addr without loading the @@ -472,7 +495,22 @@ func (up *unmarkedPutter) Put(num kv.Num, v []byte) error { } func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd, tx} + return &temporalGetter{sd: sd, tx: tx, wm: sd.mainWM} +} + +// AsGetterNoMetrics returns a getter whose GetLatest collects no metrics. Use +// for concurrent callers that share this SD but don't yet have per-worker +// metrics wiring (parallel exec workers): metering them into sd.mainWM would be +// a data race. Their per-worker metering is a follow-up (the state-read path). +func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { + return &temporalGetter{sd: sd, tx: tx, wm: nil} +} + +// MergeWorkerMetrics folds a worker's lock-free accumulator into the shared +// DomainMetrics under a single lock. Called once when a worker finishes (not +// per read), so the global metrics write-lock stays off the hot path. +func (sd *SharedDomains) MergeWorkerMetrics(wm *changeset.WorkerMetrics) { + sd.metrics.Merge(wm) } // LockChangesetAccumulator and UnlockChangesetAccumulator bracket a @@ -580,14 +618,21 @@ func (sd *SharedDomains) GetDiffset(tx kv.RwTx, blockHash common.Hash, blockNumb func (sd *SharedDomains) Unwind(txNumUnwindTo uint64, changeset *[kv.DomainLen][]kv.DomainEntryDiff) { sd.mem.Unwind(txNumUnwindTo, changeset) - // After unwind the canonical commitment-domain values are rolled - // back; any cached entries for keys touched in the unwind window - // now hold stale bytes vs the post-unwind canonical state. Drop - // those specifically — the rest of the cache (including pinned - // branches whose keys weren't in the changeset) stays warm. - if sd.branchCache != nil && changeset != nil { - for _, diff := range changeset[kv.CommitmentDomain] { - sd.branchCache.Invalidate([]byte(diff.Key)) + // Tx-aware unwind of the commitment BranchCache: every cached branch + // whose bytes belong to the rolled-back window (txNum above the unwind + // point, current epoch) is now stale vs the post-unwind canonical state. + // The changeset-gated Invalidate below only covers keys the unwound + // blocks explicitly wrote — it misses entries seeded by the read-pop and + // the trunk preload, which is what left stale committed branches that a + // fork-validation then read as a wrong trie root. Unwind(txNum) drops all + // of them (lazily, by epoch+floor) while keeping at/below-floor entries + // — including frozen preloads — warm. + if sd.branchCache != nil { + sd.branchCache.Unwind(txNumUnwindTo) + if changeset != nil { + for _, diff := range changeset[kv.CommitmentDomain] { + sd.branchCache.Invalidate([]byte(diff.Key)) + } } } } @@ -796,7 +841,7 @@ func (sd *SharedDomains) Flush(ctx context.Context, tx kv.RwTx) error { if len(v) == 0 { sd.branchCache.Invalidate(k) } else { - sd.branchCache.Put(k, v, txNum/sd.StepSize(), "sd.Flush") + sd.branchCache.Put(k, v, txNum, "sd.Flush") } } case kv.AccountsDomain: @@ -918,8 +963,39 @@ func (sd *SharedDomains) ClearBranchCache() { } } -// TemporalDomain satisfaction +// DetachBranchCache makes this SharedDomains ignore the aggregator-scope +// BranchCache: commitment branch reads go straight to sd.mem/overlay/MDBX and +// no read populates the shared cache. Used for fork-validation SDs, which read +// transient fork state — sharing the canonical BranchCache let them read stale +// committed branches (wrong trie root → INVALID payload) and pollute the cache +// with fork-transient branches. Only sd.branchCache (the read/populate path, +// domain_shared GetLatest) is consulted, so nil-ing it fully detaches; the +// canonical SDs keep their warm cache. +func (sd *SharedDomains) DetachBranchCache() { + sd.branchCache = nil +} + +// TemporalDomain satisfaction. Reads on behalf of the main goroutine, recording +// metrics into the shared lock-free main accumulator. func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { + return sd.getLatestMetered(domain, tx, k, sd.mainWM) +} + +// GetLatestContext is the context-aware read for callers that read on behalf of +// a concurrent worker: metrics go to the per-worker, lock-free accumulator +// carried by ctx (nil ctx-value => no metrics). Lets a worker's reader meter +// without touching sd.mainWM (a race) or any lock. Mirrors +// temporalGetter.GetLatestContext for readers that hold the SD directly (e.g. +// the committer's asOfStateReader). +func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { + return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) +} + +// getLatestMetered is the read implementation. wm is the caller's lock-free +// metrics accumulator (the main goroutine's sd.mainWM, or a warmup worker's +// private one); a nil wm disables metrics for the call. No global metrics lock +// is taken on this hot path — accumulators are combined later via Merge. +func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") } @@ -934,7 +1010,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // so the value is already accessible without caching it again. if v, step, ok := sd.mem.GetLatest(domain, k); ok { if dbg.KVReadLevelledMetrics { - sd.metrics.UpdateCacheReads(domain, start) + wm.UpdateCacheReads(domain, start) } return v, step, nil } else { @@ -947,7 +1023,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) if sd.parent != nil { if v, step, ok := sd.parent.mem.GetLatest(domain, k); ok { if dbg.KVReadLevelledMetrics { - sd.metrics.UpdateCacheReads(domain, start) + wm.UpdateCacheReads(domain, start) } return v, step, nil } else { @@ -958,7 +1034,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } type MeteredGetter interface { - MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) + MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } // stateCache holds in-flight values from previous transactions in the same batch @@ -967,9 +1043,9 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) v, ok := sd.stateCache.Get(domain, k) if dbg.KVReadLevelledMetrics { if ok { - sd.metrics.UpdateStateCacheHit(domain) + wm.UpdateStateCacheHit(domain) } else { - sd.metrics.UpdateStateCacheMiss(domain) + wm.UpdateStateCacheMiss(domain) } } if ok { @@ -979,7 +1055,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // backing tx is the single source of truth for this key at this point. var vDB []byte if aggTx, okAgg := tx.AggTx().(MeteredGetter); okAgg { - vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) + vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) } else { vDB, _, _ = tx.GetLatest(domain, k) } @@ -998,8 +1074,29 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // equivalent to reading from MDBX. Cleared by sd.Flush so a new MDBX // state never coexists with stale cache entries. if domain == kv.CommitmentDomain && sd.branchCache != nil { - if cv, cstep, ok := sd.branchCache.Get(k); ok { - return cv, kv.Step(cstep), nil + if cv, cTxNum, ok := sd.branchCache.Get(k); ok { + // Respect maxStep. sd.mem / sd.parent.mem lowered maxStep above when an + // unwound key reports its restored step via unwindChangeset (the per-key + // in-mem unwind signal). The cache's global epoch/floor is coarser than + // that per-key signal, so a cached entry below the floor can still belong + // to a step the unwind re-bound away. Serving it then diverges from the + // (maxStep-bounded) DB read the cache-disabled path takes. Fall through to + // the bounded read in that case; the Put below refreshes the entry. + cStep := kv.Step(cTxNum / sd.StepSize()) + if cStep <= maxStep { + if dbg.EnvBool("DBG_BC", false) { + var vDB []byte + if aggTx, okAgg := tx.AggTx().(MeteredGetter); okAgg { + vDB, _, _, _ = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) + } else { + vDB, _, _ = tx.GetLatest(domain, k) + } + if !bytes.Equal(cv, vDB) { + fmt.Fprintf(os.Stderr, "[BC-DIVERGE] key=%x cStep=%d maxStep=%d cTxNum=%d sdTxNum=%d cached=%x db=%x\n", k, cStep, maxStep, cTxNum, sd.txNum, cv, vDB) + } + } + return cv, cStep, nil + } } } @@ -1021,7 +1118,7 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } if aggTx, ok := tx.AggTx().(MeteredGetter); ok { - v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, &sd.metrics, start) + v, step, _, err = aggTx.MeteredGetLatest(domain, k, tx, maxStep, wm, start) } else { v, step, err = tx.GetLatest(domain, k) } @@ -1044,7 +1141,10 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) } } if domain == kv.CommitmentDomain && sd.branchCache != nil && len(v) > 0 { - sd.branchCache.Put(k, v, uint64(step), "sd.GetLatest") + // Stamp the upper-bound txNum of the step the bytes came from, so an + // unwind into a recent step drops this entry (frozen-step reads, with + // a low txNum, stay warm). Mirrors the stateCache read-pop above. + sd.branchCache.Put(k, v, (uint64(step)+1)*sd.StepSize()-1, "sd.GetLatest") } return v, step, nil @@ -1178,13 +1278,28 @@ func decodeAccountCodeHash(enc []byte) []byte { return h[:] } +// flushMainWM folds the main goroutine's lock-free accumulator into the shared +// DomainMetrics and resets it, so a subsequent read of sd.metrics reflects the +// main goroutine's reads. Must be called from the main goroutine (the sole +// writer of mainWM). Warmup workers merge their own accumulators independently +// via MergeWorkerMetrics; both serialize on sd.metrics' lock. +func (sd *SharedDomains) flushMainWM() { + if sd.mainWM == nil { + return + } + sd.metrics.Merge(sd.mainWM) + sd.mainWM.Reset() +} + func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { + sd.flushMainWM() return &sd.metrics } func (sd *SharedDomains) LogMetrics() []any { var metrics []any + sd.flushMainWM() sd.metrics.RLock() defer sd.metrics.RUnlock() diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 9edf294d242..302914e6cee 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -19,6 +19,8 @@ package commitment import ( "bytes" "fmt" + "math" + "os" "sync" "sync/atomic" "time" @@ -225,6 +227,18 @@ type BranchCache struct { // exactly once per cache lifetime, even though many SharedDomains // instances may be created over a process's lifetime. preloadClaimed atomic.Bool + + // Tx-aware unwind coherence — same discipline as the state-cache + // GenericCache. Each entry carries the txNum its bytes are valid as of + // and the epoch it was written in. Unwind(txNum) bumps the epoch and + // lowers unwindFloor; an entry is valid iff it was written in the + // current epoch OR its txNum is at/below the floor. Without this the + // cache cannot be unwound: a reorg leaves recent canonical branches + // that a later fork-validation reads as a wrong trie root. Step + // granularity is too coarse here — an unwind to a txNum inside the + // latest step needs txNum precision. + epoch atomic.Uint32 + unwindFloor atomic.Uint64 } // TryClaimPreload returns true the first time it's called on a given @@ -264,12 +278,18 @@ type branchCacheEntry struct { writeSeq uint64 writeTimeNanos int64 - // step is the on-disk file step the cached bytes came from. Returned - // by Get so callers (e.g. CheckDataAvailable) can validate against - // the latest visible step. 0 means "step not tracked" — fine for - // in-memory tests but real callers should always pass the step - // returned by aggTx.MeteredGetLatest / tx.GetLatest. - step uint64 + // txNum is an upper bound on the txNum the cached bytes are valid as of + // (the last txNum of the file/step they came from). Used to gate reads + // after an unwind: an entry whose txNum is above the unwind floor and + // whose epoch is superseded is stale. 0 means frozen/untracked — always + // at/below any unwind floor, so kept (correct for frozen-file preloads, + // which can never be unwound into). Real callers pass the value's txNum. + txNum uint64 + + // epoch is the unwind generation the entry was written in. Disambiguates + // a txNum reused across forks: an entry from a superseded epoch with a + // txNum above the floor is dropped lazily on its next Get. + epoch uint32 } // DefaultBranchCacheTailCapacity is the LRU tail size used when no @@ -300,10 +320,14 @@ func NewBranchCache(tailCapacity int) *BranchCache { if err != nil { panic(fmt.Sprintf("BranchCache: NewLRU: %s", err)) } - return &BranchCache{ + bc := &BranchCache{ tail: tail, pinned: maphash.NewMap[*branchCacheEntry](), } + // No unwind seen yet: every entry's txNum is at/below the floor, so the + // epoch check never strands a valid entry. + bc.unwindFloor.Store(math.MaxUint64) + return bc } // isRootPrefix reports whether prefix targets the pinned root slot. The @@ -372,7 +396,7 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // eager preload of hot prefixes — e.g. the storage-trunk of big // contracts under PIN_CONTRACT_TRUNKS. Data is copied; safe to mutate // the input after the call. -func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin string) { +func (c *BranchCache) PinEntry(prefix []byte, data []byte, txNum uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -380,7 +404,8 @@ func (c *BranchCache) PinEntry(prefix []byte, data []byte, step uint64, origin s copy(dataCopy, data) c.pinned.Set(prefix, &branchCacheEntry{ data: dataCopy, - step: step, + txNum: txNum, + epoch: c.epoch.Load(), writeSeq: c.writeSeq.Add(1), origin: origin, writeTimeNanos: time.Now().UnixNano(), @@ -440,8 +465,47 @@ func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { if !ok { return nil, 0, false } + // Tx-aware unwind invalidation: an entry from a superseded epoch whose + // txNum is above the unwind floor is stale (its bytes belong to a chain + // segment that was unwound). Drop it lazily on read. Entries at/below + // the floor (incl. frozen-file preloads stamped txNum 0) stay warm. + if entry.epoch != c.epoch.Load() && entry.txNum > c.unwindFloor.Load() { + c.Invalidate(prefix) + if dbgBC { + fmt.Fprintf(os.Stderr, "[BC-EVICT] prefix=%x txNum=%d floor=%d eEpoch=%d cur=%d\n", prefix, entry.txNum, c.unwindFloor.Load(), entry.epoch, c.epoch.Load()) + } + return nil, 0, false + } + if dbgBC && entry.epoch != c.epoch.Load() { + fmt.Fprintf(os.Stderr, "[BC-SERVE-OLD] prefix=%x txNum=%d floor=%d eEpoch=%d cur=%d\n", prefix, entry.txNum, c.unwindFloor.Load(), entry.epoch, c.epoch.Load()) + } c.bytesServed.Add(uint64(len(entry.data))) - return entry.data, entry.step, true + return entry.data, entry.txNum, true +} + +var dbgBC = os.Getenv("DBG_BC") != "" + +// Unwind invalidates cache entries whose bytes belong to a chain segment +// above unwindToTxNum. O(1): bump the epoch (so entries written in the new, +// live epoch stay valid) and lower the floor to the unwind point (so entries +// at/below it survive); stale entries (superseded epoch, txNum above the +// floor) are dropped lazily on their next Get. Mirrors GenericCache.Unwind. +// Driven from SharedDomains.Unwind so a reorg can't leave stale committed +// branches that a fork-validation then reads as a wrong trie root. +func (c *BranchCache) Unwind(unwindToTxNum uint64) { + c.epoch.Add(1) + if dbgBC { + fmt.Fprintf(os.Stderr, "[BC-UNWIND] txNum=%d newEpoch=%d\n", unwindToTxNum, c.epoch.Load()) + } + for { + cur := c.unwindFloor.Load() + if unwindToTxNum >= cur { + break + } + if c.unwindFloor.CompareAndSwap(cur, unwindToTxNum) { + break + } + } } // GetDecoded retrieves the cached branch in decoded form. Lazy-decodes on @@ -489,7 +553,7 @@ func (c *BranchCache) GetDecoded(prefix []byte) (bitmap uint16, cells *[16]cell, // caller buffer lifetime. step is the on-disk file step the bytes came // from (0 if not tracked); origin is a short label of the write site // captured for divergence-detection diagnostics. -func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string) { +func (c *BranchCache) Put(prefix []byte, data []byte, txNum uint64, origin string) { if isCommitmentStateKey(prefix) { return } @@ -497,7 +561,8 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string copy(dataCopy, data) c.store(prefix, &branchCacheEntry{ data: dataCopy, - step: step, + txNum: txNum, + epoch: c.epoch.Load(), origin: origin, writeSeq: c.writeSeq.Add(1), writeTimeNanos: time.Now().UnixNano(), @@ -511,11 +576,11 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step uint64, origin string // // Same semantics as WarmupCache.PutBranchIfClean — see that doc for the // race-it-protects-against narrative. -func (c *BranchCache) PutIfClean(prefix []byte, data []byte, step uint64, origin string) bool { +func (c *BranchCache) PutIfClean(prefix []byte, data []byte, txNum uint64, origin string) bool { if existing, ok := c.lookup(prefix); ok && existing.dirty.Load() { return false } - c.Put(prefix, data, step, origin) + c.Put(prefix, data, txNum, origin) return true } @@ -584,6 +649,7 @@ func (c *BranchCache) Clear() { c.root.Store(nil) c.pinned = maphash.NewMap[*branchCacheEntry]() c.tail.Purge() + c.unwindFloor.Store(math.MaxUint64) c.rootHits.Store(0) c.rootMisses.Store(0) c.pinnedHits.Store(0) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 874c5f651fa..ed62f5769fd 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -38,6 +38,9 @@ type sd interface { SetTxNum(blockNum uint64) AsGetter(tx kv.TemporalTx) kv.TemporalGetter AsPutDel(tx kv.TemporalTx) kv.TemporalPutDel + // MergeWorkerMetrics folds a finished worker's lock-free metrics accumulator + // into the shared metrics (once, not per read). + MergeWorkerMetrics(wm *changeset.WorkerMetrics) StepSize() uint64 Trace() bool CommitmentCapture() bool @@ -500,6 +503,14 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex if err != nil { return &errorTrieContext{err: err}, func() {} } + // Warmup workers run concurrently with the main commitment goroutine. + // Each gets its own lock-free metrics accumulator carried in a per-worker + // context, so reads neither share metrics state with the main goroutine + // (a data race) nor take the global metrics write lock (the prior hard + // serialization point). The accumulator is folded into the shared metrics + // once at teardown. + wm := changeset.NewWorkerMetrics() + workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -507,11 +518,12 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex txNum: txNum, } if sdc.stateReader != nil { - warmupCtx.stateReader = sdc.stateReader.Clone(roTx) + warmupCtx.stateReader = sdc.stateReader.CloneForWorker(workerCtx, roTx) } else { - warmupCtx.stateReader = NewLatestStateReader(roTx, sdc.sharedDomains) + warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { + sdc.sharedDomains.MergeWorkerMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup @@ -539,6 +551,11 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont collectors = append(collectors, collector) mu.Unlock() + // Concurrent mounts run in parallel; each gets its own lock-free metrics + // accumulator via a per-worker context so they don't share metrics state + // (a race) or take the global metrics lock. Folded in at teardown. + wm := changeset.NewWorkerMetrics() + workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -547,11 +564,12 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont localCollector: collector, } if sdc.stateReader != nil { - warmupCtx.stateReader = sdc.stateReader.Clone(roTx) + warmupCtx.stateReader = sdc.stateReader.CloneForWorker(workerCtx, roTx) } else { - warmupCtx.stateReader = NewLatestStateReader(roTx, sdc.sharedDomains) + warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { + sdc.sharedDomains.MergeWorkerMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup diff --git a/execution/commitment/commitmentdb/commitment_context_test.go b/execution/commitment/commitmentdb/commitment_context_test.go index 868443018e1..11d04181f11 100644 --- a/execution/commitment/commitmentdb/commitment_context_test.go +++ b/execution/commitment/commitmentdb/commitment_context_test.go @@ -1,6 +1,7 @@ package commitmentdb import ( + "context" "math/rand" "testing" @@ -55,6 +56,8 @@ func (r *testStateReader) Read(d kv.Domain, key []byte, stepSize uint64) ([]byte func (r *testStateReader) Clone(kv.TemporalTx) StateReader { return r } +func (r *testStateReader) CloneForWorker(context.Context, kv.TemporalTx) StateReader { return r } + func Test_TrieContext_BranchCopiesData(t *testing.T) { t.Parallel() diff --git a/execution/commitment/commitmentdb/reader.go b/execution/commitment/commitmentdb/reader.go index 805931727db..fc9500c424d 100644 --- a/execution/commitment/commitmentdb/reader.go +++ b/execution/commitment/commitmentdb/reader.go @@ -1,6 +1,7 @@ package commitmentdb import ( + "context" "fmt" "github.com/erigontech/erigon/db/kv" @@ -11,18 +12,43 @@ type StateReader interface { CheckDataAvailable(d kv.Domain, step kv.Step) error Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) Clone(tx kv.TemporalTx) StateReader + // CloneForWorker clones the reader for a concurrent worker (trie-warmup / + // concurrent-commitment mount). Behaves like Clone except reads are metered + // into the per-worker accumulator carried by workerCtx, so concurrent + // workers never touch the main goroutine's lock-free accumulator (a race) + // or take the global metrics lock. + CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader +} + +// ctxGetter is the optional context-aware read method (see +// temporalGetter.GetLatestContext). Worker readers type-assert for it so reads +// meter into the per-worker accumulator carried by the worker context. +type ctxGetter interface { + GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) } type LatestStateReader struct { sharedDomains sd getter kv.TemporalGetter srcTx kv.TemporalTx + // workerCtx, when non-nil, carries this worker's lock-free metrics + // accumulator; reads route through getter.GetLatestContext(workerCtx, …) so + // concurrent workers don't share the main accumulator. Nil on the main + // reader, which meters into sd.mainWM via the plain GetLatest. + workerCtx context.Context } func NewLatestStateReader(tx kv.TemporalTx, sd sd) *LatestStateReader { return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx} } +// NewLatestStateReaderForWorker is like NewLatestStateReader but reads meter +// into the per-worker accumulator carried by workerCtx (for concurrent +// workers). See LatestStateReader.workerCtx. +func NewLatestStateReaderForWorker(workerCtx context.Context, tx kv.TemporalTx, sd sd) *LatestStateReader { + return &LatestStateReader{sharedDomains: sd, getter: sd.AsGetter(tx), srcTx: tx, workerCtx: workerCtx} +} + func (r *LatestStateReader) WithHistory() bool { return false } @@ -36,6 +62,15 @@ func (r *LatestStateReader) CheckDataAvailable(d kv.Domain, step kv.Step) error } func (r *LatestStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) { + if r.workerCtx != nil { + if cg, ok := r.getter.(ctxGetter); ok { + enc, step, err = cg.GetLatestContext(r.workerCtx, d, plainKey) + if err != nil { + return nil, 0, fmt.Errorf("LatestStateReader(GetLatestContext) %q: %w", d, err) + } + return enc, step, nil + } + } enc, step, err = r.getter.GetLatest(d, plainKey) if err != nil { return nil, 0, fmt.Errorf("LatestStateReader(GetLatest) %q: %w", d, err) @@ -51,7 +86,13 @@ func (r *LatestStateReader) Clone(tx kv.TemporalTx) StateReader { // Before flush drained sd.mem this was masked because the in-memory batch // still held the source values; rebinding sd.AsGetter to the foreign compute // tx reads the wrong database and yields empty state (wrong root). - return NewLatestStateReader(r.srcTx, r.sharedDomains) + return &LatestStateReader{sharedDomains: r.sharedDomains, getter: r.sharedDomains.AsGetter(r.srcTx), srcTx: r.srcTx, workerCtx: r.workerCtx} +} + +// CloneForWorker clones into a worker reader that meters into workerCtx's +// per-worker accumulator. Source tx preserved, same as Clone. +func (r *LatestStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return NewLatestStateReaderForWorker(workerCtx, r.srcTx, r.sharedDomains) } // HistoryStateReader reads *full* historical state at specified txNum. @@ -88,6 +129,12 @@ func (r *HistoryStateReader) Clone(tx kv.TemporalTx) StateReader { return NewHistoryStateReader(tx, r.limitReadAsOfTxNum) } +// CloneForWorker: history reads go straight to roTx.GetAsOf (no shared metrics +// accumulator), so it's identical to Clone. +func (r *HistoryStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) StateReader { + return NewHistoryStateReader(tx, r.limitReadAsOfTxNum) +} + // FilesOnlyStateReader reads from .kv files only, capped at limitTxNum. // On miss (key not present in any frozen .kv file ≤ limitTxNum), returns nil // without any fallback. This is the right semantic for integrity checks and @@ -123,6 +170,12 @@ func (r *FilesOnlyStateReader) Clone(tx kv.TemporalTx) StateReader { return NewFilesOnlyStateReader(tx, r.limitTxNum) } +// CloneForWorker: files-only reads go straight to the tx (no shared metrics +// accumulator), so it's identical to Clone. +func (r *FilesOnlyStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) StateReader { + return NewFilesOnlyStateReader(tx, r.limitTxNum) +} + // SplitStateReader implements commitmentdb.StateReader using (potentially) different state readers for commitment // data and account/storage/code data. type SplitStateReader struct { @@ -163,6 +216,13 @@ func (r *SplitStateReader) Clone(tx kv.TemporalTx) StateReader { return NewCommitmentSplitStateReader(r.commitmentReader.Clone(tx), r.plainStateReader.Clone(tx), r.withHistory) } +// CloneForWorker propagates the worker clone to sub-readers so an embedded +// LatestStateReader (the commitment reader) meters into the per-worker +// accumulator instead of the shared one. +func (r *SplitStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return NewCommitmentSplitStateReader(r.commitmentReader.CloneForWorker(workerCtx, tx), r.plainStateReader.CloneForWorker(workerCtx, tx), r.withHistory) +} + func NewCommitmentSplitStateReader(commitmentReader StateReader, plainStateReader StateReader, withHistory bool) *SplitStateReader { return &SplitStateReader{ commitmentReader: commitmentReader, @@ -197,6 +257,18 @@ func (crsr *CommitmentReplayStateReader) Clone(tx kv.TemporalTx) StateReader { } } +// CloneForWorker mirrors Clone but meters the commitment (Latest) reader into +// the per-worker accumulator carried by workerCtx. +func (crsr *CommitmentReplayStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return &CommitmentReplayStateReader{ + SplitStateReader: NewCommitmentSplitStateReader( + crsr.commitmentReader.CloneForWorker(workerCtx, tx), + crsr.plainStateReader, + false, + ), + } +} + // RebuildStateReader creates a StateReader for building commitment from scratch, block-by-block. // Commitment is read from SharedDomains' in-memory batch (LatestStateReader) because we are generating // it incrementally - prior commitment state lives in the MemBatch, not yet on disk. @@ -241,3 +313,14 @@ func (r *RebuildStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) func (r *RebuildStateReader) Clone(tx kv.TemporalTx) StateReader { return NewRebuildStateReader(tx, r.sd, r.plainStateAsOf) } + +// CloneForWorker mirrors Clone but the commitment (Latest) reader meters into +// the per-worker accumulator carried by workerCtx. +func (r *RebuildStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) StateReader { + return &RebuildStateReader{ + commitmentReader: NewLatestStateReaderForWorker(workerCtx, tx, r.sd), + plainStateReader: NewHistoryStateReader(tx, r.plainStateAsOf), + plainStateAsOf: r.plainStateAsOf, + sd: r.sd, + } +} diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 88fa7cc2677..7d0584b5050 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -124,7 +124,11 @@ func (p *ContractTrunkPreload) Run( break } - cache.PinEntry(prefix, v, step, "preload-trunk") + // Trunk preload reads frozen .kv files only (below any unwind point), + // so the bytes can never go stale on a reorg — stamp txNum 0 ("frozen, + // always valid") so the tx-aware Get keeps them warm across unwinds. + _ = step + cache.PinEntry(prefix, v, 0, "preload-trunk") // Copy prefix because nibbles.HexToCompact may return a slice // over a reused buffer; we need a stable handle for later // Invalidate on demotion. diff --git a/execution/exec/state.go b/execution/exec/state.go index 1fc34bc96e3..d519578b0f0 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -226,7 +226,7 @@ func (rw *Worker) ResetState(rs *state.StateV3Buffered, chainTx kv.TemporalTx, s } else { var getter kv.TemporalGetter if chainTx != nil { - getter = rs.Domains().AsGetter(chainTx) + getter = rs.Domains().AsGetterNoMetrics(chainTx) } // Use CachedReaderV3 for parallel workers — caches account data // on first read per block, providing a stable pre-block committed @@ -292,7 +292,7 @@ func (rw *Worker) resetTx(chainTx kv.TemporalTx) error { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetter(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) case historic: typedReader.SetTx(rw.chainTx) default: @@ -415,7 +415,7 @@ func (rw *Worker) SetReader(reader state.StateReader) { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetter(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) case historic: typedReader.SetTx(rw.chainTx) } @@ -447,7 +447,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask Task) *TxResult { // the coinbase race investigation). rw.SetReader(state.NewHistoryReaderV3WithSharedDomains(rw.chainTx, rw.rs.Domains(), txTask.Version().TxNum)) } else if !txTask.IsHistoric() && (rw.stateReader == nil || rw.historyMode) { - rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetter(rw.chainTx), nil)) + rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx), nil)) } // Set the per-block committed state cache from the task. @@ -533,7 +533,7 @@ func NewWorkersPool(ctx context.Context, accumulator *shards.Accumulator, backgr reader := stateReader if reader == nil { - reader = state.NewReaderV3(rs.Domains().AsGetter(nil)) + reader = state.NewReaderV3(rs.Domains().AsGetterNoMetrics(nil)) } if err = reconWorkers[i].ResetState(rs, nil, reader, stateWriter, accumulator); err != nil { diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 532c5f29355..5767cfae91b 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -403,6 +403,8 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa err = fmt.Errorf("updateForkChoice: %w", err) return sendForkchoiceErrorWithoutWaiting(e.logger, outcomeCh, err, false) } + // SD.Unwind (inside RunUnwind) tx-aware-invalidates the BranchCache by + // the unwound txNum, so no whole-cache clear is needed here. UpdateForkChoiceDepth(fcuHeader.Number.Uint64() - 1 - unwindTarget) diff --git a/execution/stagedsync/committer.go b/execution/stagedsync/committer.go index 56e6c5fde18..e2900758b2b 100644 --- a/execution/stagedsync/committer.go +++ b/execution/stagedsync/committer.go @@ -528,6 +528,11 @@ type asOfStateReader struct { sd *execctx.SharedDomains roTx kv.TemporalTx txNum uint64 + // workerCtx, when non-nil, carries this worker's lock-free metrics + // accumulator; the CommitmentDomain read routes through GetLatestContext so + // a concurrent trie-warmup worker doesn't write the shared main accumulator + // (a race) or take the global metrics lock. Nil on the main reader. + workerCtx context.Context } func (r *asOfStateReader) WithHistory() bool { return false } @@ -539,7 +544,11 @@ func (r *asOfStateReader) CheckDataAvailable(d kv.Domain, step kv.Step) error { func (r *asOfStateReader) Read(d kv.Domain, plainKey []byte, stepSize uint64) (enc []byte, step kv.Step, err error) { if d == kv.CommitmentDomain { // Branches: use GetLatest — written only by this calculator, sequential. - enc, step, err = r.sd.GetLatest(d, r.roTx, plainKey) + if r.workerCtx != nil { + enc, step, err = r.sd.GetLatestContext(r.workerCtx, d, r.roTx, plainKey) + } else { + enc, step, err = r.sd.GetLatest(d, r.roTx, plainKey) + } } else { // Account/storage/code: use GetAsOf to avoid reading future state. // Check sd.mem first (in-memory data from current batch), then @@ -570,5 +579,13 @@ func (r *asOfStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader { return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum} } +// CloneForWorker meters the worker's CommitmentDomain reads into the per-worker +// accumulator carried by workerCtx (this reader is used as the commitment +// reader during block assembly, where trie-warmup runs concurrently — so it +// must not write the shared main accumulator). +func (r *asOfStateReader) CloneForWorker(workerCtx context.Context, tx kv.TemporalTx) commitmentdb.StateReader { + return &asOfStateReader{sd: r.sd, roTx: tx, txNum: r.txNum, workerCtx: workerCtx} +} + // Keep imports used. var _ = commitment.CommitProgress{} diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 1eca858a1f2..7e1bf7c6424 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -2540,7 +2540,7 @@ func (be *blockExecutor) nextResult(ctx context.Context, pe *parallelExecutor, r // BlockStateCache write buffer. This ensures the // system TX sees all accumulated state from prior // TXs in the block, not stale sd.mem values. - stateReader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetter(applyTx), be.blockStateCache) + stateReader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetterNoMetrics(applyTx), be.blockStateCache) } } @@ -2803,7 +2803,7 @@ func (be *blockExecutor) nextResult(ctx context.Context, pe *parallelExecutor, r // the pre-block balance and stomped tx 28's in-block update. reader = state.NewHistoryReaderV3WithBlockCache(applyTx, pe.rs.Domains(), be.blockStateCache, finalVersion.TxNum) } else { - reader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetter(applyTx), be.blockStateCache) + reader = state.NewCurrentCachedReaderV3(pe.rs.Domains().AsGetterNoMetrics(applyTx), be.blockStateCache) } pe.RUnlock() diff --git a/rpc/jsonrpc/eth_simulation.go b/rpc/jsonrpc/eth_simulation.go index 3a377700d39..948650d07f5 100644 --- a/rpc/jsonrpc/eth_simulation.go +++ b/rpc/jsonrpc/eth_simulation.go @@ -999,6 +999,13 @@ func (r *simulationStateReader) Clone(tx kv.TemporalTx) commitmentdb.StateReader return newHistoryCommitmentOnlyReader(tx, r.sd, r.commitmentAsOfTxNum, r.plainStateAsOfTxNum) } +// CloneForWorker mirrors Clone. eth_simulation runs commitment single-threaded +// (no concurrent warmup), so worker metering isn't needed here; deferred to the +// state-read-path migration. +func (r *simulationStateReader) CloneForWorker(_ context.Context, tx kv.TemporalTx) commitmentdb.StateReader { + return newHistoryCommitmentOnlyReader(tx, r.sd, r.commitmentAsOfTxNum, r.plainStateAsOfTxNum) +} + func newHistoryCommitmentOnlyReader(roTx kv.TemporalTx, sd *execctx.SharedDomains, commitmentAsOfTxNum uint64, plainStateAsOfTxNum uint64) commitmentdb.StateReader { return &simulationStateReader{sd: sd, roTx: roTx, commitmentAsOfTxNum: commitmentAsOfTxNum, plainStateAsOfTxNum: plainStateAsOfTxNum} } From a0270020108c18a1604c7617500000591d58d9c8 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 20:25:33 +0000 Subject: [PATCH 115/120] db/state: drop process-wide metrics accumulator (metrics-on safety) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit kept a single SharedDomains.mainWM that every AsGetter caller recorded into. But AsGetter is used by many concurrent goroutines on a live node (RPC, engine API, txpool, exec) — so a single lock-free accumulator was both raced and unbounded (its uniqueSeen map grew without limit and was written from every read). With KV_READ_METRICS=true this degraded a long-running node to a crawl (cancun 225/226 failing on insufficient-funds / timeouts); metrics-off was unaffected since the accumulator was never touched. Remove mainWM (and flushMainWM): GetLatest now collects no metrics. Read metrics are gathered only per-task / per-worker via GetLatestContext (the commitment + trie-warmup readers), each accumulator bounded to one task and merged into the shared DomainMetrics at teardown. AsGetterNoMetrics is kept as an explicit-intent alias. Wiring the exec/RPC read paths onto per-task contexts is a follow-up. Verified: KV_READ_METRICS=true cancun back to the 3 known failures (was 225); -race + metrics-on clean; build/gofmt clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/execctx/domain_shared.go | 62 ++++++++++--------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 073ecf3b0d9..6464a7dd057 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -109,11 +109,6 @@ type SharedDomains struct { disableInlineTouchKey bool mem kv.TemporalMemBatch metrics changeset.DomainMetrics - // mainWM is the lock-free metrics accumulator for the main goroutine's - // reads (direct GetLatest + the main commitment/exec getter). Trie-warmup - // workers get their own via AsGetterWorker; all are combined into metrics - // at task boundaries. Keeps the global metrics write-lock off the hot path. - mainWM *changeset.WorkerMetrics // blockOverlay is an in-memory overlay for block-level metadata writes (headers, bodies, // canonical hashes, TD, stage progress, forkchoice markers). It allows execution to @@ -203,7 +198,6 @@ func NewSharedDomainsWithTrieVariant(ctx context.Context, tx kv.TemporalTx, logg logger: logger, //trace: true, metrics: changeset.DomainMetrics{Domains: map[kv.Domain]*changeset.DomainIOMetrics{}}, - mainWM: changeset.NewWorkerMetrics(), stepSize: tx.Debug().StepSize(), } @@ -440,15 +434,16 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx - // wm is the accumulator GetLatest records into: sd.mainWM for the main - // goroutine (AsGetter), or nil to collect nothing (AsGetterNoMetrics) for - // concurrent callers whose per-worker metering isn't wired yet (parallel - // exec workers) — they must not write the shared main accumulator (a race). - wm *changeset.WorkerMetrics } +// GetLatest collects no read metrics. There is no process-wide accumulator: +// AsGetter is used by many concurrent goroutines (RPC, engine, exec) and a +// shared lock-free accumulator would be raced/unbounded. Metrics are collected +// only per-task via GetLatestContext (the commitment/warmup worker readers), +// merged into the shared DomainMetrics at task end. Wiring the exec/RPC read +// paths onto per-task contexts is a follow-up. func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, gt.wm) + return gt.sd.getLatestMetered(name, gt.tx, k, nil) } // GetLatestContext is the context-aware read: it records into the per-worker, @@ -495,15 +490,14 @@ func (up *unmarkedPutter) Put(num kv.Num, v []byte) error { } func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd: sd, tx: tx, wm: sd.mainWM} + return &temporalGetter{sd: sd, tx: tx} } -// AsGetterNoMetrics returns a getter whose GetLatest collects no metrics. Use -// for concurrent callers that share this SD but don't yet have per-worker -// metrics wiring (parallel exec workers): metering them into sd.mainWM would be -// a data race. Their per-worker metering is a follow-up (the state-read path). +// AsGetterNoMetrics is retained as an explicit-intent alias of AsGetter (which +// already collects no metrics) for concurrent callers (parallel exec workers) +// where the no-metrics choice is deliberate pending per-task wiring. func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { - return &temporalGetter{sd: sd, tx: tx, wm: nil} + return &temporalGetter{sd: sd, tx: tx} } // MergeWorkerMetrics folds a worker's lock-free accumulator into the shared @@ -975,26 +969,25 @@ func (sd *SharedDomains) DetachBranchCache() { sd.branchCache = nil } -// TemporalDomain satisfaction. Reads on behalf of the main goroutine, recording -// metrics into the shared lock-free main accumulator. +// TemporalDomain satisfaction. Collects no read metrics — see +// temporalGetter.GetLatest for why there is no process-wide accumulator. func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { - return sd.getLatestMetered(domain, tx, k, sd.mainWM) + return sd.getLatestMetered(domain, tx, k, nil) } // GetLatestContext is the context-aware read for callers that read on behalf of // a concurrent worker: metrics go to the per-worker, lock-free accumulator // carried by ctx (nil ctx-value => no metrics). Lets a worker's reader meter -// without touching sd.mainWM (a race) or any lock. Mirrors -// temporalGetter.GetLatestContext for readers that hold the SD directly (e.g. -// the committer's asOfStateReader). +// without any shared accumulator or lock. Mirrors temporalGetter.GetLatestContext +// for readers that hold the SD directly (e.g. the committer's asOfStateReader). func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) } // getLatestMetered is the read implementation. wm is the caller's lock-free -// metrics accumulator (the main goroutine's sd.mainWM, or a warmup worker's -// private one); a nil wm disables metrics for the call. No global metrics lock -// is taken on this hot path — accumulators are combined later via Merge. +// per-task/per-worker metrics accumulator (nil disables metrics for the call). +// No global metrics lock is taken on this hot path — accumulators are combined +// into the shared DomainMetrics later via Merge. func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") @@ -1278,28 +1271,13 @@ func decodeAccountCodeHash(enc []byte) []byte { return h[:] } -// flushMainWM folds the main goroutine's lock-free accumulator into the shared -// DomainMetrics and resets it, so a subsequent read of sd.metrics reflects the -// main goroutine's reads. Must be called from the main goroutine (the sole -// writer of mainWM). Warmup workers merge their own accumulators independently -// via MergeWorkerMetrics; both serialize on sd.metrics' lock. -func (sd *SharedDomains) flushMainWM() { - if sd.mainWM == nil { - return - } - sd.metrics.Merge(sd.mainWM) - sd.mainWM.Reset() -} - func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { - sd.flushMainWM() return &sd.metrics } func (sd *SharedDomains) LogMetrics() []any { var metrics []any - sd.flushMainWM() sd.metrics.RLock() defer sd.metrics.RUnlock() From 943968dde6b83d45e0e3f050d4f1a09d4108e29f Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 21:06:15 +0000 Subject: [PATCH 116/120] commitment: meter the main-fold reads per-ComputeCommitment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metrics-on safety fix left the main (root-fold) commitment reads unmetered — only trie-warmup workers collected, giving a partial read profile. Give the main fold its own per-ComputeCommitment lock-free accumulator (carried via the read context, merged into the shared metrics when commitment finishes). The fold is single-goroutine so it owns the accumulator exclusively; warmup / concurrent-mount workers keep their own. Now KV_READ_METRICS captures the full commitment read profile without any process-wide accumulator. Verified: -race + KV_READ_METRICS=true clean (3x) on execmodule parallel block-assembly; build/gofmt clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../commitmentdb/commitment_context.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index ed62f5769fd..fe082edd1df 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -207,7 +207,11 @@ func NewSharedDomainsCommitmentContext(sd sd, mode commitment.Mode, trieVariant return ctx } -func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNum, txNum uint64) *TrieContext { +// trieContext builds the main (root-fold) trie read context. readCtx carries +// the per-ComputeCommitment lock-free metrics accumulator (nil-value => no +// metrics); the main fold is single-goroutine so it owns that accumulator +// exclusively. Warmup/concurrent-mount readers get their own via the factories. +func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNum, txNum uint64, readCtx context.Context) *TrieContext { mainTtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(tx), putter: sdc.sharedDomains.AsPutDel(tx), @@ -218,9 +222,9 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu probeTx: tx, } if sdc.stateReader != nil { - mainTtx.stateReader = sdc.stateReader.Clone(tx) + mainTtx.stateReader = sdc.stateReader.CloneForWorker(readCtx, tx) } else { - mainTtx.stateReader = NewLatestStateReader(tx, sdc.sharedDomains) + mainTtx.stateReader = NewLatestStateReaderForWorker(readCtx, tx, sdc.sharedDomains) } sdc.patriciaTrie.ResetContext(mainTtx) return mainTtx @@ -346,7 +350,15 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context } } - trieContext := sdc.trieContext(tx, blockNum, txNum) + // Per-ComputeCommitment metrics accumulator for the main (root-fold) reads. + // The fold is single-goroutine, so this lock-free accumulator is owned + // exclusively here; merged into the shared metrics when commitment finishes. + // Warmup/concurrent-mount workers accumulate + merge independently. + commitMetrics := changeset.NewWorkerMetrics() + defer sdc.sharedDomains.MergeWorkerMetrics(commitMetrics) + readCtx := changeset.ContextWithWorkerMetrics(ctx, commitMetrics) + + trieContext := sdc.trieContext(tx, blockNum, txNum, readCtx) // If trie trace is configured, wrap the context with a recorder. // Block-targeted: when TrieTraceBlock is set, only record that specific block. @@ -659,7 +671,7 @@ func (sdc *SharedDomainsCommitmentContext) enableConcurrentCommitmentIfPossible( // SeekCommitment searches for last encoded state from DomainCommitted // and if state found, sets it up to current domain func (sdc *SharedDomainsCommitmentContext) SeekCommitment(ctx context.Context, tx kv.TemporalTx) (txNum, blockNum uint64, err error) { - trieContext := sdc.trieContext(tx, 0, 0) // blockNum/txNum not yet known; trieContext only used for reading here + trieContext := sdc.trieContext(tx, 0, 0, ctx) // blockNum/txNum not yet known; trieContext only used for reading here _, _, state, err := sdc.LatestCommitmentState(trieContext) if err != nil { From f375aa842784ee4220ce23d513477fbc1a45c68e Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 22:55:20 +0000 Subject: [PATCH 117/120] db/state, commitment: collapse metrics to per-worker DomainMetrics instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces changeset.WorkerMetrics (added in 44355fd) with plain per-worker *DomainMetrics instances — removing both the name collision with the unrelated execution/exec.WorkerMetrics and an extra type. The model is now: - Update* are lock-free and run on a single-owner per-worker instance (one per trie-warmup worker, one per ComputeCommitment for the main fold). - Merge folds a finished worker instance into the shared aggregate under the aggregate's lock — the only protected write. A separate goroutine reading the aggregate under RLock always sees a consistent snapshot. So: updates unprotected, aggregation/snapshot protected — no per-read lock, no atomics, no process-wide accumulator. Also drops the UniqueFileReadCount / UniqueLenBuckets read-amplification tracking: it had no consumer (only a comment referenced it) and cost a string-alloc + map-insert on every file read — a needless hot-path cost. The struct fields remain (unpopulated) to avoid churn. Update* gain nil receiver guards: a nil *DomainMetrics is the valid "collect no metrics" sentinel for reads with no per-worker context (e.g. SeekCommitment, plain AsGetter on RPC/exec paths). Verified: build, make lint (0 issues), gofmt, -race + KV_READ_METRICS=true clean (3x), and hive cancun with KV_READ_METRICS=true back to the 3 known fails (node healthy). Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/aggregator.go | 4 +- db/state/changeset/state_changeset.go | 284 ++++++------------ db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 12 +- .../commitmentdb/commitment_context.go | 22 +- 5 files changed, 112 insertions(+), 212 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index c83c8568dbd..387e4f4a906 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -2478,11 +2478,11 @@ func (at *AggregatorRoTx) GetLatest(domain kv.Domain, k []byte, tx kv.Tx) (v []b return at.getLatest(domain, k, tx, math.MaxUint64, nil, time.Time{}) } -func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { return at.getLatest(domain, k, tx, maxStep, metrics, start) } -func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { +func (at *AggregatorRoTx) getLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) { if domain != kv.CommitmentDomain { return at.d[domain].getLatest(k, tx, maxStep, metrics, start) } diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 60bbd70a3d4..c6b9e0962b7 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -501,190 +501,126 @@ type DomainIOMetrics struct { StateCacheMissCount int64 } +// DomainMetrics is used in two roles. As the shared aggregate, it is read for a +// snapshot under RLock and written only by Merge under Lock — so a separate +// goroutine always sees a consistent snapshot. As a per-worker instance (one +// per goroutine, via NewDomainMetrics), its Update* run lock-free because the +// instance is single-owner; the worker's counts are folded into the aggregate +// via Merge when the worker's task finishes. The embedded mutex therefore +// guards aggregation/snapshot only — never the per-read Update* path. type DomainMetrics struct { sync.RWMutex DomainIOMetrics Domains map[kv.Domain]*DomainIOMetrics - - // seenFileReads is a process-cumulative dedup set for - // UpdateFileReadsUnique. Keys are (domain.String() + prefixBytes); - // LoadOrStore on each read decides whether the prefix is new and - // therefore contributes to UniqueFileReadCount. Held outside the - // mutex (sync.Map handles its own concurrency) so the unique check - // doesn't serialise with other metric updates. - seenFileReads sync.Map } -// WorkerMetrics is a per-worker, lock-free I/O metrics accumulator. Each -// goroutine on the hot read path (the main commitment/exec goroutine and each -// trie-warmup worker) owns one exclusively, so updates need no synchronization -// — no mutex, no atomics, no shared cache line. The per-worker accumulators are -// combined into the shared DomainMetrics once per task via DomainMetrics.Merge. -// -// This replaces the previous design where every read called a DomainMetrics -// method that took the global write lock (dm.Lock()); under concurrent warmup -// workers that write lock was a hard serialization point on the hottest path. -type WorkerMetrics struct { - total DomainIOMetrics - domains [kv.DomainLen]DomainIOMetrics - - // uniqueSeen records the first local sighting of each (domain,prefix) that - // fell through to a file read, so cross-worker uniqueness can be resolved - // once at Merge time (against DomainMetrics.seenFileReads) instead of - // touching a shared structure on every read. Value is the length bucket. - // Nil until the first UpdateFileReadsUnique call (workers that never read - // from files pay nothing). - uniqueSeen map[uniqueKey]int +// NewDomainMetrics returns a fresh instance (used for the aggregate and for +// per-worker instances). +func NewDomainMetrics() *DomainMetrics { + return &DomainMetrics{Domains: map[kv.Domain]*DomainIOMetrics{}} } -type uniqueKey struct { - domain kv.Domain - prefix string -} - -// NewWorkerMetrics returns a fresh lock-free accumulator. A nil *WorkerMetrics -// is valid and makes every Update* a no-op, so callers that don't collect -// metrics can pass nil. -func NewWorkerMetrics() *WorkerMetrics { return &WorkerMetrics{} } - -// ctxKey is the typed-int context-key namespace for this package (the const-int -// model used by node/app), avoiding per-key struct types. -type ctxKey int - -const ( - ckWorkerMetrics ctxKey = iota -) - -// ContextWithWorkerMetrics returns a child context carrying a per-worker, -// lock-free metrics accumulator. Concurrent workers (e.g. trie-warmup -// goroutines) each run with their own, so reads never contend on a shared -// metrics structure (and take no lock). -func ContextWithWorkerMetrics(ctx context.Context, wm *WorkerMetrics) context.Context { - return context.WithValue(ctx, ckWorkerMetrics, wm) -} - -// WorkerMetricsFromContext extracts the per-worker accumulator, or nil if none -// was attached (in which case reads collect no metrics — a valid no-op). -func WorkerMetricsFromContext(ctx context.Context) *WorkerMetrics { - if ctx == nil { - return nil +// domainEntry returns the per-domain counters, creating them on first use. +// Lock-free: only called on a single-owner per-worker instance (the per-domain +// roll-up into the aggregate happens in Merge, under the aggregate's lock). +func (dm *DomainMetrics) domainEntry(domain kv.Domain) *DomainIOMetrics { + d, ok := dm.Domains[domain] + if !ok { + d = &DomainIOMetrics{} + dm.Domains[domain] = d } - wm, _ := ctx.Value(ckWorkerMetrics).(*WorkerMetrics) - return wm + return d } -// Reset zeroes the accumulator so it can be reused after its counts have been -// merged into the shared DomainMetrics. Used for the long-lived main-goroutine -// accumulator, which is merged-and-reset on each metrics read. -func (w *WorkerMetrics) Reset() { - if w == nil { - return - } - w.total = DomainIOMetrics{} - for i := range w.domains { - w.domains[i] = DomainIOMetrics{} - } - w.uniqueSeen = nil -} - -func (w *WorkerMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { - if w == nil { +// Update* record one read into this (single-owner) instance, lock-free. Do NOT +// call them on the shared aggregate from multiple goroutines — use a per-worker +// instance and Merge it in. +func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.CacheReadCount++ - w.total.CacheReadDuration += d - w.domains[domain].CacheReadCount++ - w.domains[domain].CacheReadDuration += d + dm.CacheReadCount++ + dm.CacheReadDuration += d + de := dm.domainEntry(domain) + de.CacheReadCount++ + de.CacheReadDuration += d } -func (w *WorkerMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { - if w == nil { +func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.DbReadCount++ - w.total.DbReadDuration += d - w.domains[domain].DbReadCount++ - w.domains[domain].DbReadDuration += d + dm.DbReadCount++ + dm.DbReadDuration += d + de := dm.domainEntry(domain) + de.DbReadCount++ + de.DbReadDuration += d } -func (w *WorkerMetrics) UpdateStateCacheHit(domain kv.Domain) { - if w == nil { +func (dm *DomainMetrics) UpdateStateCacheHit(domain kv.Domain) { + if dm == nil { return } - w.total.StateCacheHitCount++ - w.domains[domain].StateCacheHitCount++ + dm.StateCacheHitCount++ + dm.domainEntry(domain).StateCacheHitCount++ } -func (w *WorkerMetrics) UpdateStateCacheMiss(domain kv.Domain) { - if w == nil { +func (dm *DomainMetrics) UpdateStateCacheMiss(domain kv.Domain) { + if dm == nil { return } - w.total.StateCacheMissCount++ - w.domains[domain].StateCacheMissCount++ + dm.StateCacheMissCount++ + dm.domainEntry(domain).StateCacheMissCount++ } -func (w *WorkerMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { - if w == nil { +func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { + if dm == nil { return } d := time.Since(start) - w.total.FileReadCount++ - w.total.FileReadDuration += d - w.domains[domain].FileReadCount++ - w.domains[domain].FileReadDuration += d + dm.FileReadCount++ + dm.FileReadDuration += d + de := dm.domainEntry(domain) + de.FileReadCount++ + de.FileReadDuration += d } -// Merge folds a worker's accumulated counters into the shared DomainMetrics -// under a single lock acquisition (once per task, not once per read). Unique -// file-read prefixes are deduplicated across workers here, against the global -// seenFileReads set, so the cross-worker uniqueness count stays correct without -// any shared structure on the hot path. -func (dm *DomainMetrics) Merge(w *WorkerMetrics) { - if w == nil { +// UpdateFileReadsUnique records a file read. The distinct-prefix (read +// amplification) tracking it used to do was removed: it had no consumer and +// cost a string alloc + map insert on every file read. Thin alias so call sites +// are unchanged. +func (dm *DomainMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { + dm.UpdateFileReads(domain, start) +} + +// Merge folds a finished per-worker instance into this aggregate under the +// aggregate's lock — the join. This and the RLock'd snapshot are the only +// protected operations. +func (dm *DomainMetrics) Merge(src *DomainMetrics) { + if src == nil { return } dm.Lock() defer dm.Unlock() - addIOMetrics(&dm.DomainIOMetrics, &w.total) - for d := range w.domains { - src := &w.domains[d] - if src.isZero() { - continue - } - dst, ok := dm.Domains[kv.Domain(d)] - if !ok { - dst = &DomainIOMetrics{} - dm.Domains[kv.Domain(d)] = dst - } - addIOMetrics(dst, src) - } - for uk, bucket := range w.uniqueSeen { - domainKey := uk.domain.String() + ":" + uk.prefix - if _, already := dm.seenFileReads.LoadOrStore(domainKey, struct{}{}); already { - continue - } - dm.UniqueFileReadCount++ - dm.UniqueLenBuckets[bucket]++ - if dst, ok := dm.Domains[uk.domain]; ok { - dst.UniqueFileReadCount++ - dst.UniqueLenBuckets[bucket]++ - } else { - nd := &DomainIOMetrics{UniqueFileReadCount: 1} - nd.UniqueLenBuckets[bucket] = 1 - dm.Domains[uk.domain] = nd - } + addIOMetrics(&dm.DomainIOMetrics, &src.DomainIOMetrics) + for d, sd := range src.Domains { + addIOMetrics(dm.domainEntry(d), sd) } } -// addIOMetrics adds src's cumulative read-side counters into dst. CachePut* -// fields are intentionally excluded: those track mem-batch buffer occupancy -// (functional state, reset on flush), not cumulative read metrics, and are -// accounted separately off this path. func addIOMetrics(dst, src *DomainIOMetrics) { dst.CacheReadCount += src.CacheReadCount dst.CacheReadDuration += src.CacheReadDuration + dst.CacheGetCount += src.CacheGetCount + dst.CachePutCount += src.CachePutCount + dst.CacheGetSize += src.CacheGetSize + dst.CacheGetKeySize += src.CacheGetKeySize + dst.CacheGetValueSize += src.CacheGetValueSize + dst.CachePutSize += src.CachePutSize + dst.CachePutKeySize += src.CachePutKeySize + dst.CachePutValueSize += src.CachePutValueSize dst.DbReadCount += src.DbReadCount dst.DbReadDuration += src.DbReadDuration dst.FileReadCount += src.FileReadCount @@ -693,60 +629,24 @@ func addIOMetrics(dst, src *DomainIOMetrics) { dst.StateCacheMissCount += src.StateCacheMissCount } -func (m *DomainIOMetrics) isZero() bool { - return m.CacheReadCount == 0 && m.DbReadCount == 0 && m.FileReadCount == 0 && - m.StateCacheHitCount == 0 && m.StateCacheMissCount == 0 -} +// ctxKey is this package's typed-int context-key namespace (the node/app model). +type ctxKey int -// lenBucket maps a prefix byte-length to its UniqueLenBuckets index. -// See DomainIOMetrics.UniqueLenBuckets for the bucket layout. Buckets -// 5-8 sub-divide the storage subtree (depths 64-127) so per-contract -// trunk vs deep leaf-parent reads are distinguishable. -func lenBucket(n int) int { - switch { - case n <= 1: - return 0 - case n <= 4: - return 1 - case n <= 8: - return 2 - case n <= 16: - return 3 - case n <= 32: - return 4 - case n == 33: - return 5 - case n <= 36: - return 6 - case n <= 44: - return 7 - case n <= 64: - return 8 - default: - return 9 - } -} +const ckMetrics ctxKey = iota -// UpdateFileReadsUnique records a file read in the worker-local accumulator, -// remembering the (domain,prefix) first-sighting so cross-worker uniqueness can -// be resolved at Merge time. Lock-free: the key bytes are copied into a -// per-worker map; nothing shared is touched on the hot path. The key bytes are -// copied; do not mutate after the call. -func (w *WorkerMetrics) UpdateFileReadsUnique(domain kv.Domain, key []byte, start time.Time) { - if w == nil { - return - } - d := time.Since(start) - w.total.FileReadCount++ - w.total.FileReadDuration += d - w.domains[domain].FileReadCount++ - w.domains[domain].FileReadDuration += d +// ContextWithMetrics attaches a per-worker metrics instance to ctx; the read +// path extracts it via MetricsFromContext so a concurrent worker accumulates +// into its own instance and never touches shared state. +func ContextWithMetrics(ctx context.Context, m *DomainMetrics) context.Context { + return context.WithValue(ctx, ckMetrics, m) +} - if w.uniqueSeen == nil { - w.uniqueSeen = make(map[uniqueKey]int) - } - uk := uniqueKey{domain: domain, prefix: string(key)} - if _, ok := w.uniqueSeen[uk]; !ok { - w.uniqueSeen[uk] = lenBucket(len(key)) +// MetricsFromContext returns the per-worker instance, or nil (reads then collect +// no metrics — a valid no-op). +func MetricsFromContext(ctx context.Context) *DomainMetrics { + if ctx == nil { + return nil } + m, _ := ctx.Value(ckMetrics).(*DomainMetrics) + return m } diff --git a/db/state/domain.go b/db/state/domain.go index e512287659e..d9158f5ce42 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -1611,7 +1611,7 @@ func (dt *DomainRoTx) GetLatest(key []byte, roTx kv.Tx) ([]byte, kv.Step, bool, return dt.getLatest(key, roTx, math.MaxInt64, nil, time.Time{}) } -func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) ([]byte, kv.Step, bool, error) { +func (dt *DomainRoTx) getLatest(key []byte, roTx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) ([]byte, kv.Step, bool, error) { if dt.d.Disable { return nil, 0, false, nil } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 6464a7dd057..16d2d3a1186 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -454,7 +454,7 @@ func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv // AggregatorRoTx.MeteredGetLatest pattern). The plan is to migrate GetLatest // callers onto this ctx-carrying form in a follow-up, then drop the split. func (gt *temporalGetter) GetLatestContext(ctx context.Context, name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, changeset.WorkerMetricsFromContext(ctx)) + return gt.sd.getLatestMetered(name, gt.tx, k, changeset.MetricsFromContext(ctx)) } // GetCodeSize returns the length of the code at addr without loading the @@ -500,10 +500,10 @@ func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } -// MergeWorkerMetrics folds a worker's lock-free accumulator into the shared +// MergeMetrics folds a worker's lock-free accumulator into the shared // DomainMetrics under a single lock. Called once when a worker finishes (not // per read), so the global metrics write-lock stays off the hot path. -func (sd *SharedDomains) MergeWorkerMetrics(wm *changeset.WorkerMetrics) { +func (sd *SharedDomains) MergeMetrics(wm *changeset.DomainMetrics) { sd.metrics.Merge(wm) } @@ -981,14 +981,14 @@ func (sd *SharedDomains) GetLatest(domain kv.Domain, tx kv.TemporalTx, k []byte) // without any shared accumulator or lock. Mirrors temporalGetter.GetLatestContext // for readers that hold the SD directly (e.g. the committer's asOfStateReader). func (sd *SharedDomains) GetLatestContext(ctx context.Context, domain kv.Domain, tx kv.TemporalTx, k []byte) (v []byte, step kv.Step, err error) { - return sd.getLatestMetered(domain, tx, k, changeset.WorkerMetricsFromContext(ctx)) + return sd.getLatestMetered(domain, tx, k, changeset.MetricsFromContext(ctx)) } // getLatestMetered is the read implementation. wm is the caller's lock-free // per-task/per-worker metrics accumulator (nil disables metrics for the call). // No global metrics lock is taken on this hot path — accumulators are combined // into the shared DomainMetrics later via Merge. -func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.WorkerMetrics) (v []byte, step kv.Step, err error) { +func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k []byte, wm *changeset.DomainMetrics) (v []byte, step kv.Step, err error) { if tx == nil { return nil, 0, errors.New("sd.GetLatest: unexpected nil tx") } @@ -1027,7 +1027,7 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k } type MeteredGetter interface { - MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.WorkerMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) + MeteredGetLatest(domain kv.Domain, k []byte, tx kv.Tx, maxStep kv.Step, metrics *changeset.DomainMetrics, start time.Time) (v []byte, step kv.Step, ok bool, err error) } // stateCache holds in-flight values from previous transactions in the same batch diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index fe082edd1df..791a26450f4 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -38,9 +38,9 @@ type sd interface { SetTxNum(blockNum uint64) AsGetter(tx kv.TemporalTx) kv.TemporalGetter AsPutDel(tx kv.TemporalTx) kv.TemporalPutDel - // MergeWorkerMetrics folds a finished worker's lock-free metrics accumulator + // MergeMetrics folds a finished worker's lock-free metrics accumulator // into the shared metrics (once, not per read). - MergeWorkerMetrics(wm *changeset.WorkerMetrics) + MergeMetrics(wm *changeset.DomainMetrics) StepSize() uint64 Trace() bool CommitmentCapture() bool @@ -354,9 +354,9 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context // The fold is single-goroutine, so this lock-free accumulator is owned // exclusively here; merged into the shared metrics when commitment finishes. // Warmup/concurrent-mount workers accumulate + merge independently. - commitMetrics := changeset.NewWorkerMetrics() - defer sdc.sharedDomains.MergeWorkerMetrics(commitMetrics) - readCtx := changeset.ContextWithWorkerMetrics(ctx, commitMetrics) + commitMetrics := changeset.NewDomainMetrics() + defer sdc.sharedDomains.MergeMetrics(commitMetrics) + readCtx := changeset.ContextWithMetrics(ctx, commitMetrics) trieContext := sdc.trieContext(tx, blockNum, txNum, readCtx) @@ -521,8 +521,8 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex // (a data race) nor take the global metrics write lock (the prior hard // serialization point). The accumulator is folded into the shared metrics // once at teardown. - wm := changeset.NewWorkerMetrics() - workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) + wm := changeset.NewDomainMetrics() + workerCtx := changeset.ContextWithMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -535,7 +535,7 @@ func (sdc *SharedDomainsCommitmentContext) trieContextFactory(ctx context.Contex warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { - sdc.sharedDomains.MergeWorkerMetrics(wm) + sdc.sharedDomains.MergeMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup @@ -566,8 +566,8 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont // Concurrent mounts run in parallel; each gets its own lock-free metrics // accumulator via a per-worker context so they don't share metrics state // (a race) or take the global metrics lock. Folded in at teardown. - wm := changeset.NewWorkerMetrics() - workerCtx := changeset.ContextWithWorkerMetrics(ctx, wm) + wm := changeset.NewDomainMetrics() + workerCtx := changeset.ContextWithMetrics(ctx, wm) warmupCtx := &TrieContext{ getter: sdc.sharedDomains.AsGetter(roTx), putter: sdc.sharedDomains.AsPutDel(roTx), @@ -581,7 +581,7 @@ func (sdc *SharedDomainsCommitmentContext) concurrentTrieContextFactory(ctx cont warmupCtx.stateReader = NewLatestStateReaderForWorker(workerCtx, roTx, sdc.sharedDomains) } cleanup := func() { - sdc.sharedDomains.MergeWorkerMetrics(wm) + sdc.sharedDomains.MergeMetrics(wm) roTx.Rollback() } return warmupCtx, cleanup From cd1fb8114f496593fab563bb467919a790d3e907 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sat, 6 Jun 2026 23:09:38 +0000 Subject: [PATCH 118/120] exec: meter parallel-exec worker reads (per-worker DomainMetrics, per-task merge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Un-defers the exec-path read metrics that the metrics refactor had left collecting nothing (AsGetterNoMetrics), which made DomainMetrics undercount on a syncing node (exec reads dominate). Each exec Worker now owns a private *changeset.DomainMetrics; its state reader records into it via AsGetterMetered (lock-free, single-owner). The instance is merged into the shared SharedDomains metrics and reset once per task — right after the result is queued, off the hot path. That replaces a lock-per-read with a lock-per-task (a task has many reads), so contention is far lower; skipped entirely when KV_READ_METRICS is off. So exec3_metrics' per-domain Account/Storage/Code cache-layer profile is now populated, complementing exec.WorkerMetrics' per-worker read counts. Verified: build, make lint (0 issues), gofmt, -race + KV_READ_METRICS=true clean. Hive metrics-on validation pending (next). Co-Authored-By: Claude Opus 4.8 (1M context) --- db/state/changeset/state_changeset.go | 13 +++++++++++++ db/state/execctx/domain_shared.go | 27 ++++++++++++++++---------- execution/exec/state.go | 28 ++++++++++++++++++++------- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index c6b9e0962b7..48ae1dc0260 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -520,6 +520,19 @@ func NewDomainMetrics() *DomainMetrics { return &DomainMetrics{Domains: map[kv.Domain]*DomainIOMetrics{}} } +// Reset zeroes a per-worker instance after its counts have been merged into the +// aggregate, so it can be reused for the next task. Lock-free: called by the +// single owning worker. The Domains map is cleared but kept allocated. +func (dm *DomainMetrics) Reset() { + if dm == nil { + return + } + dm.DomainIOMetrics = DomainIOMetrics{} + for k := range dm.Domains { + delete(dm.Domains, k) + } +} + // domainEntry returns the per-domain counters, creating them on first use. // Lock-free: only called on a single-owner per-worker instance (the per-domain // roll-up into the aggregate happens in Merge, under the aggregate's lock). diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 16d2d3a1186..61fcb201cae 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -434,16 +434,16 @@ func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k type temporalGetter struct { sd *SharedDomains tx kv.TemporalTx + // m is an optional per-worker metrics instance to record reads into. nil + // (the AsGetter default) collects nothing — there is no process-wide + // accumulator, since AsGetter is used by many concurrent goroutines (RPC, + // engine) where a shared one would be raced/unbounded. Exec workers pass + // their own instance via AsGetterMetered and merge it at task end. + m *changeset.DomainMetrics } -// GetLatest collects no read metrics. There is no process-wide accumulator: -// AsGetter is used by many concurrent goroutines (RPC, engine, exec) and a -// shared lock-free accumulator would be raced/unbounded. Metrics are collected -// only per-task via GetLatestContext (the commitment/warmup worker readers), -// merged into the shared DomainMetrics at task end. Wiring the exec/RPC read -// paths onto per-task contexts is a follow-up. func (gt *temporalGetter) GetLatest(name kv.Domain, k []byte) (v []byte, step kv.Step, err error) { - return gt.sd.getLatestMetered(name, gt.tx, k, nil) + return gt.sd.getLatestMetered(name, gt.tx, k, gt.m) } // GetLatestContext is the context-aware read: it records into the per-worker, @@ -493,13 +493,20 @@ func (sd *SharedDomains) AsGetter(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } -// AsGetterNoMetrics is retained as an explicit-intent alias of AsGetter (which -// already collects no metrics) for concurrent callers (parallel exec workers) -// where the no-metrics choice is deliberate pending per-task wiring. +// AsGetterNoMetrics is an explicit-intent alias of AsGetter (collects no +// metrics), for concurrent callers (RPC/engine) where that is deliberate. func (sd *SharedDomains) AsGetterNoMetrics(tx kv.TemporalTx) kv.TemporalGetter { return &temporalGetter{sd: sd, tx: tx} } +// AsGetterMetered returns a getter that records reads into the caller's own +// per-worker metrics instance m. m must be single-owner (one goroutine); the +// caller merges it into the shared metrics via MergeMetrics at task end (a lock +// per task, not per read) and Resets it. Used by parallel-exec workers. +func (sd *SharedDomains) AsGetterMetered(tx kv.TemporalTx, m *changeset.DomainMetrics) kv.TemporalGetter { + return &temporalGetter{sd: sd, tx: tx, m: m} +} + // MergeMetrics folds a worker's lock-free accumulator into the shared // DomainMetrics under a single lock. Called once when a worker finishes (not // per read), so the global metrics write-lock stays off the hot path. diff --git a/execution/exec/state.go b/execution/exec/state.go index d519578b0f0..c24532439d4 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -32,6 +32,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol" @@ -126,6 +127,11 @@ type Worker struct { dirs datadir.Dirs metrics *WorkerMetrics + // readMetrics is this worker's private domain read-metrics accumulator + // (lock-free, single-owner). The worker's state reader records into it; it + // is merged into the shared SharedDomains metrics + reset once per task (a + // lock per task, not per read), at the point the result is queued. + readMetrics *changeset.DomainMetrics } // installWorkerGetHash replaces the EVM's GetHash function with one that @@ -177,8 +183,9 @@ func NewWorker(ctx context.Context, background bool, metrics *WorkerMetrics, cha evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - dirs: dirs, - metrics: metrics, + dirs: dirs, + metrics: metrics, + readMetrics: changeset.NewDomainMetrics(), } w.runnable.Store(true) w.ibs = state.New(w.stateReader) @@ -226,7 +233,7 @@ func (rw *Worker) ResetState(rs *state.StateV3Buffered, chainTx kv.TemporalTx, s } else { var getter kv.TemporalGetter if chainTx != nil { - getter = rs.Domains().AsGetterNoMetrics(chainTx) + getter = rs.Domains().AsGetterMetered(chainTx, rw.readMetrics) } // Use CachedReaderV3 for parallel workers — caches account data // on first read per block, providing a stable pre-block committed @@ -292,7 +299,7 @@ func (rw *Worker) resetTx(chainTx kv.TemporalTx) error { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics)) case historic: typedReader.SetTx(rw.chainTx) default: @@ -361,6 +368,13 @@ func (rw *Worker) Run() (err error) { if err := rw.results.Add(rw.ctx, result); err != nil { return err } + // Fold this task's reads into the shared metrics and reset. Off the hot + // path (the result is already queued): a single lock per task replacing + // the old lock-per-read. Skipped entirely when read metrics are off. + if dbg.KVReadLevelledMetrics && rw.rs != nil { + rw.rs.Domains().MergeMetrics(rw.readMetrics) + rw.readMetrics.Reset() + } } return nil } @@ -415,7 +429,7 @@ func (rw *Worker) SetReader(reader state.StateReader) { switch typedReader := rw.stateReader.(type) { case latest: - typedReader.SetGetter(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx)) + typedReader.SetGetter(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics)) case historic: typedReader.SetTx(rw.chainTx) } @@ -447,7 +461,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask Task) *TxResult { // the coinbase race investigation). rw.SetReader(state.NewHistoryReaderV3WithSharedDomains(rw.chainTx, rw.rs.Domains(), txTask.Version().TxNum)) } else if !txTask.IsHistoric() && (rw.stateReader == nil || rw.historyMode) { - rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterNoMetrics(rw.chainTx), nil)) + rw.SetReader(state.NewCachedReaderV3(rw.rs.Domains().AsGetterMetered(rw.chainTx, rw.readMetrics), nil)) } // Set the per-block committed state cache from the task. @@ -533,7 +547,7 @@ func NewWorkersPool(ctx context.Context, accumulator *shards.Accumulator, backgr reader := stateReader if reader == nil { - reader = state.NewReaderV3(rs.Domains().AsGetterNoMetrics(nil)) + reader = state.NewReaderV3(rs.Domains().AsGetterMetered(nil, reconWorkers[i].readMetrics)) } if err = reconWorkers[i].ResetState(rs, nil, reader, stateWriter, accumulator); err != nil { From bd6097ec8f61aed3a294bdd57eb72d7aa10ed70e Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sun, 7 Jun 2026 09:19:59 +0000 Subject: [PATCH 119/120] execution/cache: fix off-by-one in state-cache unwind floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state-cache unwind set its invalidation floor to unwindToTxNum = Min(UnwindPoint+1) — the first txNum of the first rolled-back block — but the staleness check dropped entries only when txNum strictly exceeded the floor (txNum > floor). An entry stamped at exactly the floor therefore survived, even though that txNum belongs to a dead (unwound) block. This is the EIP-4788 beacon-root case: the beacon-root storage write runs in a block's begin-system-tx, stamped at the block's first txNum. When that block is the first one unwound by a reorg, its txNum equals the floor, so the dead-fork value was served stale during sidechain re-execution, yielding a wrong trie root (hive engine-cancun "Sidechain Reorg"). Drop entries at-or-above the floor (txNum >= floor). The surviving block's last txNum is floor-1, so this never evicts a live entry. Add a boundary regression test (TestUnwind_EvictsEntryAtFloor) covering txNum == floor. Co-Authored-By: Claude Opus 4.8 (1M context) --- execution/cache/cache_test.go | 26 ++++++++++++++++++++++++- execution/cache/generic_cache.go | 33 +++++++++++++++++++------------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index ca83eb8b5a9..7c3bd977e13 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -689,7 +689,7 @@ func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { c.Put(below, makeValue(1), 50) // predates the unwind c.Put(above, makeValue(2), 150) // written in the unwound range - c.Unwind(100) // keep <=100, drop >100 + c.Unwind(100) // floor=100 (first unwound txNum): keep <100, drop >=100 v, ok := c.Get(below) assert.True(t, ok, "entry below the unwind point must stay warm") @@ -700,6 +700,30 @@ func TestUnwind_KeepsBelowFloor_EvictsAbove(t *testing.T) { assert.Equal(t, 1, c.Len(), "the stale entry is evicted lazily on its read") } +// Boundary regression: unwindToTxNum is the FIRST rolled-back txNum +// (Min(UnwindPoint+1)), so an entry stamped at exactly that txNum belongs to the +// first dead block and must be evicted — the drop rule is txNum>=floor, not +// txNum>floor. This reproduces the Sidechain Reorg wrong-root bug: an EIP-4788 +// beacon-root storage write runs in a block's begin-system-tx, stamped at the +// block's first txNum; when that block is the first unwound one, txNum==floor and +// a strict `>` left the dead-fork value served stale. +func TestUnwind_EvictsEntryAtFloor(t *testing.T) { + c := NewDomainCache(1 * datasize.MB) + atFloor := makeAddr(1) + belowFloor := makeAddr(2) + c.Put(atFloor, makeValue(1), 100) // first txNum of the first unwound block + c.Put(belowFloor, makeValue(2), 99) // last txNum of the surviving block + + c.Unwind(100) // floor=100, epoch->1 + + _, ok := c.Get(atFloor) + assert.False(t, ok, "entry at txNum==floor is on the dead fork and must be evicted") + + v, ok := c.Get(belowFloor) + assert.True(t, ok, "entry at txNum==floor-1 predates the unwind and must stay warm") + assert.Equal(t, makeValue(2), v) +} + // The reused-txNum case: after an unwind, the live fork re-writes a key at the // SAME txNum as the dead fork's write. The epoch — not the txNum — distinguishes // them, so the dead entry reads stale and the re-written one reads valid. diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index 0119e855d68..7280f74302f 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -60,9 +60,10 @@ type GenericCache[T any] struct { // Cache coherence across unwinds is txNum/epoch based — no block awareness, // no diffset. An entry is valid iff it was written in the current epoch OR - // its txNum is at/below unwindFloor (so it predates every unwind seen and - // can't be stale). Unwind bumps the epoch and lowers the floor (O(1), no - // scan); stale entries are dropped lazily on their next read. txNum slots + // its txNum is strictly below unwindFloor (the first unwound txNum), so it + // predates every unwind seen and can't be stale. Unwind bumps the epoch and + // lowers the floor (O(1), no scan); stale entries are dropped lazily on their + // next read. txNum slots // are reused across forks, so the epoch — not the txNum — is what tells a // dead fork's write apart from the live fork's at the same txNum. epoch atomic.Uint32 @@ -170,9 +171,14 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { return zero, false } // Lazy unwind invalidation: an entry from a superseded epoch whose txNum is - // above the unwind floor reflects dead-fork state — drop it and miss so the - // read falls through to the reverted domain and repopulates. - if e.epoch != c.epoch.Load() && e.txNum > c.unwindFloor.Load() { + // at or above the unwind floor reflects dead-fork state — drop it and miss so + // the read falls through to the reverted domain and repopulates. The floor is + // the first unwound txNum (Min(UnwindPoint+1), the first txNum of the first + // rolled-back block), so an entry stamped exactly at the floor belongs to a + // dead block — e.g. an EIP-4788 beacon-root write in the block-begin system + // tx — and must be dropped; >= not > (the surviving block's last txNum is + // floor-1, so this never drops a live entry). + if e.epoch != c.epoch.Load() && e.txNum >= c.unwindFloor.Load() { c.data.Remove(h) c.staleEvicted.Add(1) c.misses.Add(1) @@ -234,13 +240,14 @@ func (c *GenericCache[T]) Clear() { c.currentSize.Store(0) } -// Unwind invalidates entries that reflect state above unwindToTxNum on a -// now-dead fork. O(1): bump the epoch (so entries written in the new, live -// epoch stay valid) and lower the floor to the unwind point (so entries at or -// below it — which predate the unwind and can't be stale — stay warm). Stale -// entries (old epoch, txNum above the floor) are dropped lazily on their next -// read. The floor only ever moves down, to the deepest unwind seen, so a later -// shallower unwind can't resurrect entries an earlier deeper one invalidated. +// Unwind invalidates entries that reflect dead-fork state. unwindToTxNum is the +// first rolled-back txNum (Min(UnwindPoint+1)); every entry at or above it is on +// the dead fork. O(1): bump the epoch (so entries written in the new, live epoch +// stay valid) and lower the floor to unwindToTxNum (so entries strictly below it +// — which predate the unwind and can't be stale — stay warm). Stale entries (old +// epoch, txNum >= floor) are dropped lazily on their next read. The floor only +// ever moves down, to the deepest unwind seen, so a later shallower unwind can't +// resurrect entries an earlier deeper one invalidated. func (c *GenericCache[T]) Unwind(unwindToTxNum uint64) { c.epoch.Add(1) for { From 67d0f33b552800cbc657db5036aafb6216e89ed9 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Sun, 7 Jun 2026 19:52:03 +0000 Subject: [PATCH 120/120] execution: address #21380 review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engineapitester: remove the now-orphaned EngineApiTester.ChainDB field (its only consumer — the EngineXTestRunner cache-clear — was removed in 4618add1c0) plus its now-unused kv import - stagedsync/exec3, exec3_parallel: drop comments that describe code removed from main (the warmup-cache-disable workaround), per review - execmodule: add TestValidateForkPayloadOffNonTipCanonicalBlockWithCache, ExecModuleTester coverage for the unwind/fork-payload caching path — a fork off a NON-tip canonical block exercises the partial unwind + parent-link diffset masking of the BranchCache, round-tripped in both directions. The existing side-fork test only branches at genesis (full unwind). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../engineapitester/engine_api_tester.go | 10 +-- execution/execmodule/exec_module_test.go | 73 +++++++++++++++++++ execution/stagedsync/exec3.go | 8 -- execution/stagedsync/exec3_parallel.go | 8 -- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/execution/engineapi/engineapitester/engine_api_tester.go b/execution/engineapi/engineapitester/engine_api_tester.go index 407d1583b1f..15f05b82b3f 100644 --- a/execution/engineapi/engineapitester/engine_api_tester.go +++ b/execution/engineapi/engineapitester/engine_api_tester.go @@ -39,7 +39,6 @@ import ( "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" - "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain" @@ -397,7 +396,6 @@ func InitialiseEngineApiTester(ctx context.Context, args EngineApiTesterInitArgs TxnInclusionVerifier: NewTxnInclusionVerifier(rpcApiClient), Node: ethNode, NodeKey: nodeKey, - ChainDB: ethBackend.ChainKV(), cleanup: cleanup, }, nil } @@ -428,13 +426,7 @@ type EngineApiTester struct { TxnInclusionVerifier TxnInclusionVerifier Node *node.Node NodeKey *ecdsa.PrivateKey - // ChainDB is the running backend's temporal DB. Retained on the tester so - // callers (e.g. EngineXTestRunner.Run between tests in the same group) can - // reach the aggregator's BranchCache and clear it without rebuilding the - // whole tester. The reference must NOT be Closed by callers — the tester's - // cleanup owns the lifecycle. - ChainDB kv.RwDB - cleanup *cleanupHandle + cleanup *cleanupHandle } func (eat EngineApiTester) Run(t *testing.T, test func(ctx context.Context, t *testing.T, eat EngineApiTester)) { diff --git a/execution/execmodule/exec_module_test.go b/execution/execmodule/exec_module_test.go index 1deb5178cd6..00ae54e4eb5 100644 --- a/execution/execmodule/exec_module_test.go +++ b/execution/execmodule/exec_module_test.go @@ -178,6 +178,79 @@ func TestValidateChainAndUpdateForkChoiceWithSideForksThatGoBackAndForwardInHeig require.NoError(t, err) } +// TestValidateForkPayloadOffNonTipCanonicalBlockWithCache exercises the +// unwind/fork-payload caching path in ExecModule.ValidateChain. A fork that +// branches off a NON-tip canonical block forces validateChain to +// unwindToCommonCanonical (unwinding only the canonical blocks above the branch +// point), chain the validation SharedDomains to e.currentContext so the unwound +// blocks' diffsets are reachable via the parent link, and mask the BranchCache +// with the resulting unwind set. If any of that is wrong the re-executed fork +// produces a wrong trie root, so a Success validation status is the correctness +// assertion. The existing side-fork test only branches at genesis (full unwind); +// this covers the partial-unwind round trip the ValidateChain parent-link +// comment guards. +func TestValidateForkPayloadOffNonTipCanonicalBlockWithCache(t *testing.T) { + privKey, err := crypto.GenerateKey() + require.NoError(t, err) + senderAddr := crypto.PubkeyToAddress(privKey.PublicKey) + genesis := &types.Genesis{ + Config: chain.AllProtocolChanges, + Alloc: types.GenesisAlloc{ + senderAddr: {Balance: new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)}, + }, + } + m := execmoduletester.New(t, execmoduletester.WithGenesisSpec(genesis), execmoduletester.WithKey(privKey)) + to := common.Address{0xaa} + baseFee := m.Genesis.BaseFee().Uint64() + mkTx := func(nonce, amount uint64) types.Transaction { + tx, err := types.SignTx( + types.NewTransaction(nonce, to, uint256.NewInt(amount), 50000, uint256.NewInt(baseFee), nil), + *types.LatestSignerForChainID(nil), + privKey, + ) + require.NoError(t, err) + return tx + } + + // blockgen reads the latest committed DB state, so each segment must be + // generated while the chain head sits at its intended parent. Build and + // finalize the shared prefix genesis → 1 → 2 first (nonces 0,1), each block + // carrying a state-touching txn so the BranchCache + commitment hold real + // entries. After this the head is block 2. + prefix, err := blockgen.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(uint64(i), 1_000)) + }) + require.NoError(t, err) + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, prefix.Blocks)) + block2 := prefix.Blocks[1] + + // With the head at block 2 (signer nonce 2), generate both continuations off + // block 2 before inserting either: the canonical tip (block 3, nonce 2) and a + // longer fork (blocks 3',4' at nonces 2,3) that branches off the same non-tip + // block 2. + canonicalTip, err := blockgen.GenerateChain(m.ChainConfig, block2, m.Engine, m.DB, 1, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(2, 1_000)) + }) + require.NoError(t, err) + fork, err := blockgen.GenerateChain(m.ChainConfig, block2, m.Engine, m.DB, 2, func(i int, b *blockgen.BlockGen) { + b.AddTx(mkTx(uint64(2+i), 2_000)) + }) + require.NoError(t, err) + + // Finalize the canonical tip (head → block 3). + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, canonicalTip.Blocks)) + + // Validating fork.Blocks[0] (height 3, parent = block 2) must unwind canonical + // block 3 back to block 2; fork.Blocks[1] then head-extends and the FCU reorgs + // onto the longer fork. + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, fork.Blocks)) + + // Reorg back onto the original canonical block 3 (same common ancestor, + // block 2) to exercise the BranchCache masking in the other direction: + // unwind the fork blocks and re-validate canonical block 3 off block 2. + require.NoError(t, insertValidateAndUfc1By1(t.Context(), m.ExecModule, canonicalTip.Blocks)) +} + func addTwoTxnsToPool(ctx context.Context, startingNonce uint64, t *testing.T, m *execmoduletester.ExecModuleTester, txpool txpoolproto.TxpoolServer, baseFee uint64) { tx2, err := types.SignTx(types.NewTransaction(startingNonce, common.Address{1}, uint256.NewInt(10_000), params.TxGas, uint256.NewInt(baseFee), nil), *types.LatestSignerForChainID(m.ChainConfig.ChainID), m.Key) require.NoError(t, err) diff --git a/execution/stagedsync/exec3.go b/execution/stagedsync/exec3.go index 5d2e668794c..af559768993 100644 --- a/execution/stagedsync/exec3.go +++ b/execution/stagedsync/exec3.go @@ -205,14 +205,6 @@ func ExecV3(ctx context.Context, doms.EnableParaTrieDB(cfg.db) doms.EnableTrieWarmup(true) - // DO NOT re-introduce a warmup-cache disable here without working through - // the unwind caching scenarios. main carried a short-term workaround - // (`doms.EnableWarmupCache(!isApplyingBlocks && !parallel)`) that disabled - // the trie warmuper's value cache on the parallel path to dodge a - // stale-cache wrong-root. THIS PR is the proper fix for that wrong-root: - // caching stays enabled, the BranchCache + SD chain handle multi-block - // uncommitted batches and fork validation correctly. The re-org tests are - // the gate that confirms this. doms.SetDeferCommitmentUpdates(false) // Enable deferred commitment updates for fork validation and parallel initial sync. // Deferred updates batch commitment calculations to block boundaries rather than diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index 0b5a18f89b8..77530161b86 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -217,14 +217,6 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U pe.rs.Domains().SetInMemHistoryReads(true) defer pe.rs.Domains().SetInMemHistoryReads(prevInMemHistoryReads) - // Trie warmup left enabled for the parallel path. Original disable was - // based on a calculator/warmer interaction concern that turned out to be - // overly conservative — the Warmuper's reads are independent of the - // calculator's SetUpdates call. Removing the disable produced an 8× - // throughput improvement on the perf-devnet-3 SSTORE-bloated benchmark - // (block 24358306) by letting the Warmuper pre-fetch branch data while - // EVM execution runs. See #20920 for the canonical perf measurement. - // Skip step-boundary commitment — the calculator handles this. pe.rs.StateV3.SetSkipStepBoundaryCommitment(true) defer pe.rs.StateV3.SetSkipStepBoundaryCommitment(false)