diff --git a/db/state/changeset/state_changeset.go b/db/state/changeset/state_changeset.go index 057eba818db..7ab2a115ece 100644 --- a/db/state/changeset/state_changeset.go +++ b/db/state/changeset/state_changeset.go @@ -22,6 +22,7 @@ import ( "math" "strings" "sync" + "sync/atomic" "time" "github.com/erigontech/erigon/common" @@ -449,76 +450,101 @@ func ReadLowestUnwindableBlock(tx kv.Tx) (uint64, error) { } +// DomainIOMetrics fields are atomic so concurrent readers (exec workers + the +// tip warmup pool) record without a shared mutex. Durations are nanoseconds. type DomainIOMetrics struct { + CacheReadCount atomic.Int64 + CacheReadDuration atomic.Int64 + CacheGetCount atomic.Int64 + CachePutCount atomic.Int64 + CacheGetSize atomic.Int64 + CacheGetKeySize atomic.Int64 + CacheGetValueSize atomic.Int64 + CachePutSize atomic.Int64 + CachePutKeySize atomic.Int64 + CachePutValueSize atomic.Int64 + DbReadCount atomic.Int64 + DbReadDuration atomic.Int64 + FileReadCount atomic.Int64 + FileReadDuration atomic.Int64 +} + +// DomainIOSnapshot is a plain, copyable point-in-time read of DomainIOMetrics +// for delta computation by consumers. +type DomainIOSnapshot struct { CacheReadCount int64 - CacheReadDuration time.Duration + CacheReadDuration int64 CacheGetCount int64 CachePutCount int64 - CacheGetSize int - CacheGetKeySize int - CacheGetValueSize int - CachePutSize int - CachePutKeySize int - CachePutValueSize int + CacheGetSize int64 + CacheGetKeySize int64 + CacheGetValueSize int64 + CachePutSize int64 + CachePutKeySize int64 + CachePutValueSize int64 DbReadCount int64 - DbReadDuration time.Duration + DbReadDuration int64 FileReadCount int64 - FileReadDuration time.Duration + FileReadDuration int64 +} + +func (m *DomainIOMetrics) snapshot() DomainIOSnapshot { + return DomainIOSnapshot{ + CacheReadCount: m.CacheReadCount.Load(), + CacheReadDuration: m.CacheReadDuration.Load(), + CacheGetCount: m.CacheGetCount.Load(), + CachePutCount: m.CachePutCount.Load(), + CacheGetSize: m.CacheGetSize.Load(), + CacheGetKeySize: m.CacheGetKeySize.Load(), + CacheGetValueSize: m.CacheGetValueSize.Load(), + CachePutSize: m.CachePutSize.Load(), + CachePutKeySize: m.CachePutKeySize.Load(), + CachePutValueSize: m.CachePutValueSize.Load(), + DbReadCount: m.DbReadCount.Load(), + DbReadDuration: m.DbReadDuration.Load(), + FileReadCount: m.FileReadCount.Load(), + FileReadDuration: m.FileReadDuration.Load(), + } } type DomainMetrics struct { - sync.RWMutex - DomainIOMetrics - Domains map[kv.Domain]*DomainIOMetrics + DomainIOMetrics // aggregate across all domains + Domains [kv.DomainLen]DomainIOMetrics // per-domain, indexed by kv.Domain } -func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.CacheReadCount++ - readDuration := time.Since(start) - dm.CacheReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.CacheReadCount++ - d.CacheReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - CacheReadCount: 1, - CacheReadDuration: readDuration, - } +type DomainMetricsSnapshot struct { + DomainIOSnapshot + Domains [kv.DomainLen]DomainIOSnapshot +} + +func (dm *DomainMetrics) Snapshot() DomainMetricsSnapshot { + s := DomainMetricsSnapshot{DomainIOSnapshot: dm.DomainIOMetrics.snapshot()} + for i := range dm.Domains { + s.Domains[i] = dm.Domains[i].snapshot() } + return s +} + +func (dm *DomainMetrics) UpdateCacheReads(domain kv.Domain, start time.Time) { + d := int64(time.Since(start)) + dm.CacheReadCount.Add(1) + dm.CacheReadDuration.Add(d) + dm.Domains[domain].CacheReadCount.Add(1) + dm.Domains[domain].CacheReadDuration.Add(d) } func (dm *DomainMetrics) UpdateDbReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.DbReadCount++ - readDuration := time.Since(start) - dm.DbReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.DbReadCount++ - d.DbReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - DbReadCount: 1, - DbReadDuration: readDuration, - } - } + d := int64(time.Since(start)) + dm.DbReadCount.Add(1) + dm.DbReadDuration.Add(d) + dm.Domains[domain].DbReadCount.Add(1) + dm.Domains[domain].DbReadDuration.Add(d) } func (dm *DomainMetrics) UpdateFileReads(domain kv.Domain, start time.Time) { - dm.Lock() - defer dm.Unlock() - dm.FileReadCount++ - readDuration := time.Since(start) - dm.FileReadDuration += readDuration - if d, ok := dm.Domains[domain]; ok { - d.FileReadCount++ - d.FileReadDuration += readDuration - } else { - dm.Domains[domain] = &DomainIOMetrics{ - FileReadCount: 1, - FileReadDuration: readDuration, - } - } + d := int64(time.Since(start)) + dm.FileReadCount.Add(1) + dm.FileReadDuration.Add(d) + dm.Domains[domain].FileReadCount.Add(1) + dm.Domains[domain].FileReadDuration.Add(d) } diff --git a/db/state/domain.go b/db/state/domain.go index 347e322836b..9b1a24f4ef0 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -50,7 +50,6 @@ import ( "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/diagnostics/metrics" ) var ( @@ -498,16 +497,9 @@ type DomainRoTx struct { getFromFileCache *DomainGetFromFileCache } -func domainReadMetric(name kv.Domain, level int) metrics.Summary { - if level > 4 { - level = 5 - } - return mxsKVGet[name][level] -} - func (dt *DomainRoTx) getLatestFromFile(i int, filekey []byte, hi, lo uint64) (v []byte, ok bool, offset uint64, err error) { if dbg.KVReadLevelledMetrics { - defer domainReadMetric(dt.name, i).ObserveDuration(time.Now()) + defer recordKVGet(dt.name, i, time.Now()) } if dt.d.Accessors.Has(statecfg.AccessorBTree) { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 89ad7a1d0d8..40af74c252d 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -185,7 +185,6 @@ func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger, sd := &SharedDomains{ logger: logger, //trace: true, - metrics: changeset.DomainMetrics{Domains: map[kv.Domain]*changeset.DomainIOMetrics{}}, stepSize: tx.Debug().StepSize(), } @@ -767,24 +766,23 @@ func (sd *SharedDomains) Metrics() *changeset.DomainMetrics { func (sd *SharedDomains) LogMetrics() []any { var metrics []any - sd.metrics.RLock() - defer sd.metrics.RUnlock() + m := sd.metrics.Snapshot() - if readCount := sd.metrics.CacheReadCount; readCount > 0 { + if readCount := m.CacheReadCount; readCount > 0 { metrics = append(metrics, "cache", common.PrettyCounter(readCount), - "puts", common.PrettyCounter(sd.metrics.CachePutCount), + "puts", common.PrettyCounter(m.CachePutCount), "size", fmt.Sprintf("%s(%s/%s)", - common.PrettyCounter(sd.metrics.CachePutSize), common.PrettyCounter(sd.metrics.CachePutKeySize), common.PrettyCounter(sd.metrics.CachePutValueSize)), - "gets", common.PrettyCounter(sd.metrics.CacheGetCount), "size", common.PrettyCounter(sd.metrics.CacheGetSize), - "cdur", common.Round(sd.metrics.CacheReadDuration/time.Duration(readCount), 0)) + common.PrettyCounter(m.CachePutSize), common.PrettyCounter(m.CachePutKeySize), common.PrettyCounter(m.CachePutValueSize)), + "gets", common.PrettyCounter(m.CacheGetCount), "size", common.PrettyCounter(m.CacheGetSize), + "cdur", common.Round(time.Duration(m.CacheReadDuration)/time.Duration(readCount), 0)) } - if readCount := sd.metrics.DbReadCount; readCount > 0 { - metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(sd.metrics.DbReadDuration/time.Duration(readCount), 0)) + if readCount := m.DbReadCount; readCount > 0 { + metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(time.Duration(m.DbReadDuration)/time.Duration(readCount), 0)) } - if readCount := sd.metrics.FileReadCount; readCount > 0 { - metrics = append(metrics, "files", common.PrettyCounter(readCount), "fdur", common.Round(sd.metrics.FileReadDuration/time.Duration(readCount), 0)) + if readCount := m.FileReadCount; readCount > 0 { + metrics = append(metrics, "files", common.PrettyCounter(readCount), "fdur", common.Round(time.Duration(m.FileReadDuration)/time.Duration(readCount), 0)) } return metrics @@ -793,26 +791,26 @@ func (sd *SharedDomains) LogMetrics() []any { func (sd *SharedDomains) DomainLogMetrics() map[kv.Domain][]any { var logMetrics = map[kv.Domain][]any{} - sd.metrics.RLock() - defer sd.metrics.RUnlock() + snap := sd.metrics.Snapshot() - for domain, dm := range sd.metrics.Domains { + for d := range snap.Domains { + dm := snap.Domains[d] var metrics []any if readCount := dm.CacheReadCount; readCount > 0 { - metrics = append(metrics, "cache", common.PrettyCounter(readCount), "cdur", common.Round(dm.CacheReadDuration/time.Duration(readCount), 0)) + metrics = append(metrics, "cache", common.PrettyCounter(readCount), "cdur", common.Round(time.Duration(dm.CacheReadDuration)/time.Duration(readCount), 0)) } if readCount := dm.DbReadCount; readCount > 0 { - metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(dm.DbReadDuration/time.Duration(readCount), 0)) + metrics = append(metrics, "db", common.PrettyCounter(readCount), "dbdur", common.Round(time.Duration(dm.DbReadDuration)/time.Duration(readCount), 0)) } if readCount := dm.FileReadCount; readCount > 0 { - metrics = append(metrics, "files", common.PrettyCounter(readCount), "fdur", common.Round(dm.DbReadDuration/time.Duration(readCount), 0)) + metrics = append(metrics, "files", common.PrettyCounter(readCount), "fdur", common.Round(time.Duration(dm.FileReadDuration)/time.Duration(readCount), 0)) } if len(metrics) > 0 { - logMetrics[domain] = metrics + logMetrics[kv.Domain(d)] = metrics } } diff --git a/db/state/metrics.go b/db/state/metrics.go index 1b368d034e2..58df126c2b1 100644 --- a/db/state/metrics.go +++ b/db/state/metrics.go @@ -17,10 +17,32 @@ package state import ( + "sync/atomic" + "time" + "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/diagnostics/metrics" ) +// kvReadAcc is a lock-free per-(domain,level) read-time accumulator. Under the +// tip warmup pool the per-read prometheus Summary mutex serializes hundreds of +// concurrent readers; plain atomics avoid that contention. +type kvReadAcc struct { + count atomic.Int64 + nanos atomic.Int64 +} + +var mxsKVGetLF [kv.DomainLen][6]kvReadAcc + +func recordKVGet(name kv.Domain, level int, start time.Time) { + if level > 4 { + level = 5 + } + a := &mxsKVGetLF[name][level] + a.count.Add(1) + a.nanos.Add(int64(time.Since(start))) +} + var ( //LatestStateReadWarm = metrics.GetOrCreateSummary(`latest_state_read{type="warm",found="yes"}`) //nolint //LatestStateReadWarmNotFound = metrics.GetOrCreateSummary(`latest_state_read{type="warm",found="no"}`) //nolint diff --git a/db/state/temporal_mem_batch.go b/db/state/temporal_mem_batch.go index 15e4c482d73..357d96615e0 100644 --- a/db/state/temporal_mem_batch.go +++ b/db/state/temporal_mem_batch.go @@ -151,24 +151,12 @@ func (sd *TemporalMemBatch) putLatest(domain kv.Domain, key string, val []byte, defer sd.latestStateLock.Unlock() var updateMetrics = func(domain kv.Domain, putKeySize int, putValueSize int) { - sd.metrics.Lock() - defer sd.metrics.Unlock() - sd.metrics.CachePutCount++ - sd.metrics.CachePutSize += putKeySize + putValueSize - sd.metrics.CachePutKeySize += putKeySize - sd.metrics.CachePutValueSize += putValueSize - if dm, ok := sd.metrics.Domains[domain]; ok { - dm.CachePutCount++ - dm.CachePutSize += putKeySize + putValueSize - dm.CachePutKeySize += putKeySize - dm.CachePutValueSize += putValueSize - } else { - sd.metrics.Domains[domain] = &changeset.DomainIOMetrics{ - CachePutCount: 1, - CachePutSize: putKeySize + putValueSize, - CachePutKeySize: putKeySize, - CachePutValueSize: putValueSize, - } + putSize := int64(putKeySize + putValueSize) + for _, m := range []*changeset.DomainIOMetrics{&sd.metrics.DomainIOMetrics, &sd.metrics.Domains[domain]} { + m.CachePutCount.Add(1) + m.CachePutSize.Add(putSize) + m.CachePutKeySize.Add(int64(putKeySize)) + m.CachePutValueSize.Add(int64(putValueSize)) } } @@ -327,7 +315,7 @@ func (sd *TemporalMemBatch) GetAsOf(domain kv.Domain, key []byte, ts uint64) (v func (sd *TemporalMemBatch) SizeEstimate() uint64 { sd.latestStateLock.RLock() defer sd.latestStateLock.RUnlock() - return uint64(sd.metrics.CachePutSize) + return uint64(sd.metrics.CachePutSize.Load()) } func (sd *TemporalMemBatch) ClearRam() { @@ -342,18 +330,17 @@ func (sd *TemporalMemBatch) ClearRam() { sd.unwindChangeset = nil sd.unwindChangesetRaw = nil - sd.metrics.Lock() - defer sd.metrics.Unlock() - sd.metrics.CachePutCount = 0 - sd.metrics.CachePutSize = 0 - sd.metrics.CachePutKeySize = 0 - sd.metrics.CachePutValueSize = 0 - for _, dm := range sd.metrics.Domains { - dm.CachePutCount = 0 - dm.CachePutSize = 0 - dm.CachePutKeySize = 0 - dm.CachePutValueSize = 0 - } + for i := range sd.metrics.Domains { + m := &sd.metrics.Domains[i] + m.CachePutCount.Store(0) + m.CachePutSize.Store(0) + m.CachePutKeySize.Store(0) + m.CachePutValueSize.Store(0) + } + sd.metrics.CachePutCount.Store(0) + sd.metrics.CachePutSize.Store(0) + sd.metrics.CachePutKeySize.Store(0) + sd.metrics.CachePutValueSize.Store(0) } func (sd *TemporalMemBatch) IteratePrefix(domain kv.Domain, prefix []byte, roTx kv.Tx, it func(k []byte, v []byte) (cont bool, err error)) error { diff --git a/execution/stagedsync/exec3_metrics.go b/execution/stagedsync/exec3_metrics.go index 9530c8a4f47..524f34c3253 100644 --- a/execution/stagedsync/exec3_metrics.go +++ b/execution/stagedsync/exec3_metrics.go @@ -250,29 +250,21 @@ func resetDomainGauges(ctx context.Context) { } } -func updateExecDomainMetrics(metrics *changeset.DomainMetrics, prevMetrics *changeset.DomainMetrics, interval time.Duration, - executing bool) *changeset.DomainMetrics { - metrics.RLock() - defer metrics.RUnlock() - - if prevMetrics == nil { - prevMetrics = &changeset.DomainMetrics{ - Domains: map[kv.Domain]*changeset.DomainIOMetrics{}, - } - } - +func updateExecDomainMetrics(metrics *changeset.DomainMetrics, prev changeset.DomainMetricsSnapshot, interval time.Duration, + executing bool) changeset.DomainMetricsSnapshot { + cur := metrics.Snapshot() seconds := interval.Seconds() - cacheReads := metrics.CacheReadCount - prevMetrics.CacheReadCount - cacheDuration := metrics.CacheReadDuration - prevMetrics.CacheReadDuration - dbReads := metrics.DbReadCount - prevMetrics.DbReadCount - dbDuration := metrics.DbReadDuration - prevMetrics.DbReadDuration - fileReads := metrics.FileReadCount - prevMetrics.FileReadCount - fileDuration := metrics.FileReadDuration - prevMetrics.FileReadDuration - cachePutCount := metrics.CachePutCount - prevMetrics.CachePutCount - cachePutSize := metrics.CachePutSize - prevMetrics.CachePutSize - cachePutKeySize := metrics.CachePutKeySize - prevMetrics.CachePutKeySize - cachePutValueSize := metrics.CachePutValueSize - prevMetrics.CachePutValueSize + cacheReads := cur.CacheReadCount - prev.CacheReadCount + cacheDuration := cur.CacheReadDuration - prev.CacheReadDuration + dbReads := cur.DbReadCount - prev.DbReadCount + dbDuration := cur.DbReadDuration - prev.DbReadDuration + fileReads := cur.FileReadCount - prev.FileReadCount + fileDuration := cur.FileReadDuration - prev.FileReadDuration + cachePutCount := cur.CachePutCount - prev.CachePutCount + cachePutSize := cur.CachePutSize - prev.CachePutSize + cachePutKeySize := cur.CachePutKeySize - prev.CachePutKeySize + cachePutValueSize := cur.CachePutValueSize - prev.CachePutValueSize mxExecDomainReads.Set(float64(cacheReads+dbReads+fileReads) / seconds) mxExecDomainReadDuration.Set(float64(cacheDuration+dbDuration+fileDuration) / float64(cacheReads+dbReads+fileReads)) @@ -287,153 +279,103 @@ func updateExecDomainMetrics(metrics *changeset.DomainMetrics, prevMetrics *chan mxExecDomainFileReads.Set(float64(fileReads) / seconds) mxExecDomainFileReadDuration.Set(float64(fileDuration) / float64(fileReads)) - prevMetrics.DomainIOMetrics = metrics.DomainIOMetrics - - if accountMetrics, ok := metrics.Domains[kv.AccountsDomain]; ok { - var prevAccountMetrics changeset.DomainIOMetrics - - if prev, ok := prevMetrics.Domains[kv.AccountsDomain]; ok { - prevAccountMetrics = *prev - } - - cacheReads := accountMetrics.CacheReadCount - prevAccountMetrics.CacheReadCount - cacheDuration := accountMetrics.CacheReadDuration - prevAccountMetrics.CacheReadDuration - dbReads := accountMetrics.DbReadCount - prevAccountMetrics.DbReadCount - dbDuration := accountMetrics.DbReadDuration - prevAccountMetrics.DbReadDuration - fileReads := accountMetrics.FileReadCount - prevAccountMetrics.FileReadCount - fileDuration := accountMetrics.FileReadDuration - prevAccountMetrics.FileReadDuration - cachePutCount := accountMetrics.CachePutCount - prevAccountMetrics.CachePutCount - cachePutSize := accountMetrics.CachePutSize - prevAccountMetrics.CachePutSize - cachePutKeySize := accountMetrics.CachePutKeySize - prevAccountMetrics.CachePutKeySize - cachePutValueSize := accountMetrics.CachePutValueSize - prevAccountMetrics.CachePutValueSize + { + a, pa := cur.Domains[kv.AccountsDomain], prev.Domains[kv.AccountsDomain] + cacheReads := a.CacheReadCount - pa.CacheReadCount + cacheDuration := a.CacheReadDuration - pa.CacheReadDuration + dbReads := a.DbReadCount - pa.DbReadCount + dbDuration := a.DbReadDuration - pa.DbReadDuration + fileReads := a.FileReadCount - pa.FileReadCount + fileDuration := a.FileReadDuration - pa.FileReadDuration mxExecAccountDomainReads.Set(float64(cacheReads+dbReads+fileReads) / seconds) mxExecAccountDomainReadDuration.Set(float64(cacheDuration+dbDuration+fileDuration) / float64(cacheReads+dbReads+fileReads)) mxExecAccountDomainCacheReads.Set(float64(cacheReads) / seconds) mxExecAccountDomainCacheReadDuration.Set(float64(cacheDuration) / float64(cacheReads)) if executing { - mxExecAccountDomainPutRate.Set(float64(cachePutCount)) - mxExecAccountDomainPutSize.Set(float64(cachePutSize)) - mxExecAccountDomainPutKeySize.Set(float64(cachePutKeySize)) - mxExecAccountDomainPutValueSize.Set(float64(cachePutValueSize)) + mxExecAccountDomainPutRate.Set(float64(a.CachePutCount - pa.CachePutCount)) + mxExecAccountDomainPutSize.Set(float64(a.CachePutSize - pa.CachePutSize)) + mxExecAccountDomainPutKeySize.Set(float64(a.CachePutKeySize - pa.CachePutKeySize)) + mxExecAccountDomainPutValueSize.Set(float64(a.CachePutValueSize - pa.CachePutValueSize)) } mxExecAccountDomainDbReads.Set(float64(dbReads) / seconds) mxExecAccountDomainDbReadDuration.Set(float64(dbDuration) / float64(dbReads)) mxExecAccountDomainFileReads.Set(float64(fileReads) / seconds) mxExecAccountDomainFileReadDuration.Set(float64(fileDuration) / float64(fileReads)) - - prevAccountMetrics = *accountMetrics - prevMetrics.Domains[kv.AccountsDomain] = &prevAccountMetrics } - if storageMetrics, ok := metrics.Domains[kv.StorageDomain]; ok { - var prevStorageMetrics changeset.DomainIOMetrics - - if prev, ok := prevMetrics.Domains[kv.StorageDomain]; ok { - prevStorageMetrics = *prev - } - - cacheReads := storageMetrics.CacheReadCount - prevStorageMetrics.CacheReadCount - cacheDuration := storageMetrics.CacheReadDuration - prevStorageMetrics.CacheReadDuration - dbReads := storageMetrics.DbReadCount - prevStorageMetrics.DbReadCount - dbDuration := storageMetrics.DbReadDuration - prevStorageMetrics.DbReadDuration - fileReads := storageMetrics.FileReadCount - prevStorageMetrics.FileReadCount - fileDuration := storageMetrics.FileReadDuration - prevStorageMetrics.FileReadDuration - cachePutCount := storageMetrics.CachePutCount - prevStorageMetrics.CachePutCount - cachePutSize := storageMetrics.CachePutSize - prevStorageMetrics.CachePutSize - cachePutKeySize := storageMetrics.CachePutKeySize - prevStorageMetrics.CachePutKeySize - cachePutValueSize := storageMetrics.CachePutValueSize - prevStorageMetrics.CachePutValueSize + { + s, ps := cur.Domains[kv.StorageDomain], prev.Domains[kv.StorageDomain] + cacheReads := s.CacheReadCount - ps.CacheReadCount + cacheDuration := s.CacheReadDuration - ps.CacheReadDuration + dbReads := s.DbReadCount - ps.DbReadCount + dbDuration := s.DbReadDuration - ps.DbReadDuration + fileReads := s.FileReadCount - ps.FileReadCount + fileDuration := s.FileReadDuration - ps.FileReadDuration mxExecStorageDomainReads.Set(float64(cacheReads+dbReads+fileReads) / seconds) mxExecStorageDomainReadDuration.Set(float64(cacheDuration+dbDuration+fileDuration) / float64(cacheReads+dbReads+fileReads)) mxExexStorageDomainCacheReads.Set(float64(cacheReads) / seconds) mxExecStorageDomainCacheReadDuration.Set(float64(cacheDuration) / float64(cacheReads)) if executing { - mxExecStorageDomainPutRate.Set(float64(cachePutCount)) - mxExecStorageDomainPutSize.Set(float64(cachePutSize)) - mxExecStorageDomainPutKeySize.Set(float64(cachePutKeySize)) - mxExecStorageDomainPutValueSize.Set(float64(cachePutValueSize)) + mxExecStorageDomainPutRate.Set(float64(s.CachePutCount - ps.CachePutCount)) + mxExecStorageDomainPutSize.Set(float64(s.CachePutSize - ps.CachePutSize)) + mxExecStorageDomainPutKeySize.Set(float64(s.CachePutKeySize - ps.CachePutKeySize)) + mxExecStorageDomainPutValueSize.Set(float64(s.CachePutValueSize - ps.CachePutValueSize)) } mxExecStorageDomainDbReads.Set(float64(dbReads) / seconds) mxExecStorageDomainDbReadDuration.Set(float64(dbDuration) / float64(dbReads)) mxExecStorageDomainFileReads.Set(float64(fileReads) / seconds) mxExecStorageDomainFileReadDuration.Set(float64(fileDuration) / float64(fileReads)) - - prevStorageMetrics = *storageMetrics - prevMetrics.Domains[kv.StorageDomain] = &prevStorageMetrics } - if codeMetrics, ok := metrics.Domains[kv.CodeDomain]; ok && executing { - var prevCodeMetrics changeset.DomainIOMetrics - - if prev, ok := prevMetrics.Domains[kv.CodeDomain]; ok { - prevCodeMetrics = *prev - } - - cacheReads := codeMetrics.CacheReadCount - prevCodeMetrics.CacheReadCount - cacheDuration := codeMetrics.CacheReadDuration - prevCodeMetrics.CacheReadDuration - dbReads := codeMetrics.DbReadCount - prevCodeMetrics.DbReadCount - dbDuration := codeMetrics.DbReadDuration - prevCodeMetrics.DbReadDuration - fileReads := codeMetrics.FileReadCount - prevCodeMetrics.FileReadCount - fileDuration := codeMetrics.FileReadDuration - prevCodeMetrics.FileReadDuration - cachePutCount := codeMetrics.CachePutCount - prevCodeMetrics.CachePutCount - cachePutSize := codeMetrics.CachePutSize - prevCodeMetrics.CachePutSize - cachePutKeySize := codeMetrics.CachePutKeySize - prevCodeMetrics.CachePutKeySize - cachePutValueSize := codeMetrics.CachePutValueSize - prevCodeMetrics.CachePutValueSize + if executing { + c, pc := cur.Domains[kv.CodeDomain], prev.Domains[kv.CodeDomain] + cacheReads := c.CacheReadCount - pc.CacheReadCount + cacheDuration := c.CacheReadDuration - pc.CacheReadDuration + dbReads := c.DbReadCount - pc.DbReadCount + dbDuration := c.DbReadDuration - pc.DbReadDuration + fileReads := c.FileReadCount - pc.FileReadCount + fileDuration := c.FileReadDuration - pc.FileReadDuration mxExecCodeDomainReads.Set(float64(cacheReads+dbReads+fileReads) / seconds) mxExecCodeDomainReadDuration.Set(float64(cacheDuration+dbDuration+fileDuration) / float64(cacheReads+dbReads+fileReads)) mxExexCodeDomainCacheReads.Set(float64(cacheReads) / seconds) mxExecCodeDomainCacheReadDuration.Set(float64(cacheDuration) / float64(cacheReads)) - mxExecCodeDomainPutRate.Set(float64(cachePutCount)) - mxExecCodeDomainPutSize.Set(float64(cachePutSize)) - mxExecCodeDomainPutKeySize.Set(float64(cachePutKeySize)) - mxExecCodeDomainPutValueSize.Set(float64(cachePutValueSize)) + mxExecCodeDomainPutRate.Set(float64(c.CachePutCount - pc.CachePutCount)) + mxExecCodeDomainPutSize.Set(float64(c.CachePutSize - pc.CachePutSize)) + mxExecCodeDomainPutKeySize.Set(float64(c.CachePutKeySize - pc.CachePutKeySize)) + mxExecCodeDomainPutValueSize.Set(float64(c.CachePutValueSize - pc.CachePutValueSize)) mxExecCodeDomainDbReads.Set(float64(dbReads) / seconds) mxExecCodeDomainDbReadDuration.Set(float64(dbDuration) / float64(dbReads)) mxExecCodeDomainFileReads.Set(float64(fileReads) / seconds) mxExecCodeDomainFileReadDuration.Set(float64(fileDuration) / float64(fileReads)) - - prevCodeMetrics = *codeMetrics - prevMetrics.Domains[kv.CodeDomain] = &prevCodeMetrics } - if commitmentMetrics, ok := metrics.Domains[kv.CommitmentDomain]; !executing && ok { - var prevCommitmentMetrics changeset.DomainIOMetrics - - if prev, ok := prevMetrics.Domains[kv.CommitmentDomain]; ok { - prevCommitmentMetrics = *prev - } - - cacheReads := commitmentMetrics.CacheReadCount - prevCommitmentMetrics.CacheReadCount - cacheDuration := commitmentMetrics.CacheReadDuration - prevCommitmentMetrics.CacheReadDuration - dbReads := commitmentMetrics.DbReadCount - prevCommitmentMetrics.DbReadCount - dbDuration := commitmentMetrics.DbReadDuration - prevCommitmentMetrics.DbReadDuration - fileReads := commitmentMetrics.FileReadCount - prevCommitmentMetrics.FileReadCount - fileDuration := commitmentMetrics.FileReadDuration - prevCommitmentMetrics.FileReadDuration - cachePutCount := commitmentMetrics.CachePutCount - prevCommitmentMetrics.CachePutCount - cachePutSize := commitmentMetrics.CachePutSize - prevCommitmentMetrics.CachePutSize - cachePutKeySize := commitmentMetrics.CachePutKeySize - prevCommitmentMetrics.CachePutKeySize - cachePutValueSize := commitmentMetrics.CachePutValueSize - prevCommitmentMetrics.CachePutValueSize + if !executing { + c, pc := cur.Domains[kv.CommitmentDomain], prev.Domains[kv.CommitmentDomain] + cacheReads := c.CacheReadCount - pc.CacheReadCount + cacheDuration := c.CacheReadDuration - pc.CacheReadDuration + dbReads := c.DbReadCount - pc.DbReadCount + dbDuration := c.DbReadDuration - pc.DbReadDuration + fileReads := c.FileReadCount - pc.FileReadCount + fileDuration := c.FileReadDuration - pc.FileReadDuration mxCommitmentDomainReads.Set(float64(cacheReads+dbReads+fileReads) / seconds) mxCommitmentDomainReadDuration.Set(float64(cacheDuration+dbDuration+fileDuration) / float64(cacheReads+dbReads+fileReads)) mxCommitmentDomainCacheReads.Set(float64(cacheReads) / seconds) mxCommitmentDomainCacheReadDuration.Set(float64(cacheDuration) / float64(cacheReads)) - mxCommitmentDomainPutRate.Set(float64(cachePutCount)) - mxCommitmentDomainPutSize.Set(float64(cachePutSize)) - mxCommitmentDomainPutKeySize.Set(float64(cachePutKeySize)) - mxCommitmentDomainPutValueSize.Set(float64(cachePutValueSize)) + mxCommitmentDomainPutRate.Set(float64(c.CachePutCount - pc.CachePutCount)) + mxCommitmentDomainPutSize.Set(float64(c.CachePutSize - pc.CachePutSize)) + mxCommitmentDomainPutKeySize.Set(float64(c.CachePutKeySize - pc.CachePutKeySize)) + mxCommitmentDomainPutValueSize.Set(float64(c.CachePutValueSize - pc.CachePutValueSize)) mxCommitmentDomainDbReads.Set(float64(dbReads) / seconds) mxCommitmentDomainDbReadDuration.Set(float64(dbDuration) / float64(dbReads)) mxCommitmentDomainFileReads.Set(float64(fileReads) / seconds) mxCommitmentDomainFileReadDuration.Set(float64(fileDuration) / float64(fileReads)) - - prevCommitmentMetrics = *commitmentMetrics - prevMetrics.Domains[kv.CommitmentDomain] = &prevCommitmentMetrics } - return prevMetrics + return cur } func NewProgress(initialBlockNum, initialTxNum, commitThreshold uint64, updateMetrics bool, logPrefix string, logger log.Logger) *Progress { @@ -491,7 +433,7 @@ type Progress struct { prevBranchReadCount uint64 prevBranchWriteCount uint64 commitThreshold uint64 - prevDomainMetrics *changeset.DomainMetrics + prevDomainMetrics changeset.DomainMetricsSnapshot logPrefix string logger log.Logger } @@ -800,14 +742,13 @@ func (p *Progress) LogCommitments(rs *state.StateV3, ex executor, stepsInDb floa totalCacheHits := cacheBranchHits + cacheAccountHits + cacheStorageHits totalCacheMisses := missBranchCount + missAccountCount + missStorageCount - rs.Domains().Metrics().RLock() + dmSnap := rs.Domains().Metrics().Snapshot() commitVals := []any{ "bdur", common.Round(commitedBlockDur, 0), "progress", fmt.Sprintf("%s/%s", common.PrettyCounter(lastProgress.KeyIndex), common.PrettyCounter(lastProgress.UpdateCount)), - "buf", common.ByteCount(uint64(rs.Domains().Metrics().CachePutSize + rs.Domains().Metrics().CacheGetSize)), + "buf", common.ByteCount(uint64(dmSnap.CachePutSize + dmSnap.CacheGetSize)), "chit", common.PrettyCounter(totalCacheHits), "cmiss", common.PrettyCounter(totalCacheMisses), } - rs.Domains().Metrics().RUnlock() p.log("committed", suffix, te, rs, interval, te.lastCommittedBlockNum.Load(), committedDiffBlocks, te.lastCommittedTxNum.Load()-p.prevCommittedTxNum, committedTxSec, gasSec, 0, stepsInDb, commitVals)