2nd-generation caching system

core/vm/interpreter.go

@@ -132,14 +132,15 @@ type VM struct {
 }
 
 func copyJumpTable(jt *JumpTable) *JumpTable {
-	var copy JumpTable
+	copy := new(JumpTable)
+	ops := make([]operation, len(jt))
 	for i, op := range jt {
 		if op != nil {
-			opCopy := *op
-			copy[i] = &opCopy
+			ops[i] = *op
+			copy[i] = &ops[i]
 		}
 	}
-	return &copy
+	return copy
 }
 
 // NewEVMInterpreter returns a new instance of the Interpreter.

erigon-lib/kv/membatch/mapmutation_okx.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"github.com/benbjohnson/immutable"
 	"sort"
 	"strings"
 	"sync"
@@ -18,7 +19,7 @@ import (
 
 type MapmutationWithDoubleCache struct {
 	modifiedCache  map[string]map[string][]byte // table -> key -> value ie. blocks -> hash -> blockBod
-	immutableCache map[string]map[string][]byte
+	immutableCache *immutable.Map[string, *immutable.Map[string, []byte]]
 	db             kv.Tx
 	quit           <-chan struct{}
 	clean          func()
@@ -64,10 +65,12 @@ func (m *MapmutationWithDoubleCache) getMem(table string, key []byte) ([]byte, b
 		}
 	}
 
-	if _, ok := m.immutableCache[table]; !ok {
+	bucket, ok := m.immutableCache.Get(table)
+	if !ok {
 		return nil, false
 	}
-	if value, ok := m.immutableCache[table][*(*string)(unsafe.Pointer(&key))]; ok {
+
+	if value, ok := bucket.Get(*(*string)(unsafe.Pointer(&key))); ok {
 		return value, ok
 	}
 
@@ -152,15 +155,23 @@ func (m *MapmutationWithDoubleCache) Put(table string, k, v []byte) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if _, ok := m.modifiedCache[table]; !ok {
-		m.modifiedCache[table] = make(map[string][]byte)
+		m.modifiedCache[table] = make(map[string][]byte, 100)
 	}
 
 	stringKey := string(k)
 
-	var ok bool
-	if _, ok = m.modifiedCache[table][stringKey]; ok {
-		m.size += len(v) - len(m.modifiedCache[table][stringKey])
-		m.modifiedCache[table][stringKey] = v
+	if oldV, ok := m.modifiedCache[table][stringKey]; ok {
+		m.size += len(v) - len(oldV)
+
+		if cap(oldV) > len(v) {
+			oldV = oldV[:len(v)]
+			copy(oldV, v)
+
+			m.modifiedCache[table][stringKey] = oldV
+		} else {
+			m.modifiedCache[table][stringKey] = v
+		}
+
 		return nil
 	}
 	m.modifiedCache[table][stringKey] = v
@@ -191,15 +202,20 @@ func (m *MapmutationWithDoubleCache) ForEach(bucket string, fromPrefix []byte, w
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 
-	tmpCache := make(map[string]map[string][]byte, len(m.immutableCache))
-	for table, bucket := range m.immutableCache {
-		if _, ok := tmpCache[table]; !ok {
-			tmpCache[table] = make(map[string][]byte, len(bucket)*2)
+	tmpCache := make(map[string]map[string][]byte, m.immutableCache.Len())
+	outerIter := m.immutableCache.Iterator()
+	for !outerIter.Done() {
+		table, innerMap, _ := outerIter.Next()
+		innerCache := make(map[string][]byte, innerMap.Len())
+
+		// Iterate over inner map
+		innerIter := innerMap.Iterator()
+		for !innerIter.Done() {
+			key, value, _ := innerIter.Next()
+			innerCache[key] = value
 		}
 
-		for k, v := range bucket {
-			tmpCache[table][k] = v
-		}
+		tmpCache[table] = innerCache
 	}
 
 	for table, bucket := range m.modifiedCache {
@@ -322,7 +338,7 @@ func (m *MapmutationWithDoubleCache) Flush(ctx context.Context, tx kv.RwTx) erro
 	}
 
 	m.modifiedCache = map[string]map[string][]byte{}
-	m.immutableCache = map[string]map[string][]byte{}
+	m.immutableCache = immutable.NewMap[string, *immutable.Map[string, []byte]](nil)
 	m.size = 0
 	m.count = 0
 	return nil
@@ -336,7 +352,7 @@ func (m *MapmutationWithDoubleCache) Close() {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.modifiedCache = map[string]map[string][]byte{}
-	m.immutableCache = map[string]map[string][]byte{}
+	m.immutableCache = immutable.NewMap[string, *immutable.Map[string, []byte]](nil)
 	m.size = 0
 	m.count = 0
 	m.size = 0
@@ -354,7 +370,7 @@ func (m *MapmutationWithDoubleCache) panicOnEmptyDB() {
 	}
 }
 
-func (m *MapmutationWithDoubleCache) SetCache(cache map[string]map[string][]byte) {
+func (m *MapmutationWithDoubleCache) SetCache(cache *immutable.Map[string, *immutable.Map[string, []byte]]) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 

erigon-lib/kv/membatchwithdb/memory_mutation_cache.go

@@ -2,6 +2,8 @@ package membatchwithdb
 
 import (
 	"bytes"
+	"github.com/benbjohnson/immutable"
+	"sync"
 
 	"github.com/ledgerwatch/erigon-lib/common"
 
@@ -12,15 +14,15 @@ import (
 // MemoryMutationWithCache extends MemoryMutation with a caching layer
 type MemoryMutationWithCache struct {
 	*MemoryMutation
-	cache       map[string]map[string][]byte // Read-only cache, passed externally
-	modifyCache map[string]map[string][]byte // Writable cache for modifications
+	cache       *immutable.Map[string, *immutable.Map[string, []byte]] // Read-only cache, passed externally
+	modifyCache map[string]map[string][]byte                           // Writable cache for modifications
 }
 
 // NewMemoryBatchWithSizeNoSequenceWithCache creates a cached version with custom size
-func NewMemoryBatchWithSizeNoSequenceWithCache(tx kv.Tx, tmpDir string, mapSize datasize.ByteSize, cache map[string]map[string][]byte) *MemoryMutationWithCache {
+func NewMemoryBatchWithSizeNoSequenceWithCache(tx kv.Tx, tmpDir string, mapSize datasize.ByteSize, cache *immutable.Map[string, *immutable.Map[string, []byte]]) *MemoryMutationWithCache {
 	base := NewMemoryBatchWithSizeNoSequence(tx, tmpDir, mapSize)
 	if cache == nil {
-		cache = make(map[string]map[string][]byte)
+		cache = immutable.NewMap[string, *immutable.Map[string, []byte]](nil)
 	}
 
 	return &MemoryMutationWithCache{
@@ -30,9 +32,13 @@ func NewMemoryBatchWithSizeNoSequenceWithCache(tx kv.Tx, tmpDir string, mapSize
 	}
 }
 
+var keyPool = sync.Pool{New: func() interface{} { return "" }}
+
 // GetOne with cache support, prioritizes modifyCache, then cache, then MemoryMutation
 func (m *MemoryMutationWithCache) GetOne(table string, key []byte) ([]byte, error) {
-	keyStr := string(key)
+	keyStr := keyPool.Get().(string)
+	keyStr = string(key)
+	defer keyPool.Put(keyStr)
 
 	// 1. Check modifyCache first
 	if modKeys, ok := m.modifyCache[table]; ok {
@@ -42,8 +48,8 @@ func (m *MemoryMutationWithCache) GetOne(table string, key []byte) ([]byte, erro
 	}
 
 	// 2. Check read-only cache
-	if keys, ok := m.cache[table]; ok {
-		if val, exists := keys[keyStr]; exists {
+	if buckets, ok := m.cache.Get(table); ok {
+		if val, exists := buckets.Get(keyStr); exists {
 			return val, nil
 		}
 	}
@@ -54,13 +60,6 @@ func (m *MemoryMutationWithCache) GetOne(table string, key []byte) ([]byte, erro
 		return nil, err
 	}
 	_, v, err := c.SeekExact(key)
-	if err == nil && v != nil {
-		// Store in modifyCache (not cache, as cache is read-only)
-		if _, ok := m.modifyCache[table]; !ok {
-			m.modifyCache[table] = make(map[string][]byte)
-		}
-		m.modifyCache[table][keyStr] = common.Copy(v)
-	}
 	return v, err
 }
 
@@ -76,8 +75,8 @@ func (m *MemoryMutationWithCache) Has(table string, key []byte) (bool, error) {
 	}
 
 	// 2. Check read-only cache
-	if keys, ok := m.cache[table]; ok {
-		if _, exists := keys[keyStr]; exists {
+	if buckets, ok := m.cache.Get(table); ok {
+		if _, exists := buckets.Get(keyStr); exists {
 			return true, nil
 		}
 	}

eth/backend.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	debug2 "runtime/debug"
 	"slices"
 	"strconv"
 	"strings"
@@ -1372,6 +1373,8 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config, chainConfig
 	chainKv := s.chainDB
 	var err error
 
+	debug2.SetGCPercent(200)
+
 	s.stagedSync = stagedsync.New(s.config.Sync, s.syncStages, s.syncUnwindOrder, s.syncPruneOrder, s.logger)
 	// For X Layer, ac
 	if s.verifier != nil {

eth/stagedsync/stage_xlayer.go

@@ -1,10 +1,12 @@
 package stagedsync
 
-func (s *StageState) GetSmtCache() map[string]map[string][]byte {
+import "github.com/benbjohnson/immutable"
+
+func (s *StageState) GetSmtCache() *immutable.Map[string, *immutable.Map[string, []byte]] {
 	return s.state.GetSmtCache()
 }
 
-func (s *StageState) GetSmtHistorySnapshotCache(blockNumber uint64) map[string]map[string][]byte {
+func (s *StageState) GetSmtHistorySnapshotCache(blockNumber uint64) *immutable.Map[string, *immutable.Map[string, []byte]] {
 	return s.state.GetSmtSnapshotCache(blockNumber)
 }
 

eth/stagedsync/sync_xlayer.go

@@ -1,18 +1,19 @@
 package stagedsync
 
 import (
+	"github.com/benbjohnson/immutable"
 	"github.com/ledgerwatch/erigon/zk/smt"
 )
 
 func (s *Sync) GetCache() *smt.SmtCache {
 	return s.cache
 }
 
-func (s *Sync) GetSmtCache() map[string]map[string][]byte {
+func (s *Sync) GetSmtCache() *immutable.Map[string, *immutable.Map[string, []byte]] {
 	return s.cache.GetSmtCache()
 }
 
-func (s *Sync) GetSmtSnapshotCache(blockNumber uint64) map[string]map[string][]byte {
+func (s *Sync) GetSmtSnapshotCache(blockNumber uint64) *immutable.Map[string, *immutable.Map[string, []byte]] {
 	return s.cache.CascadeGetCurrentBatchSnapshotCache(blockNumber)
 }
 

go.mod

@@ -159,7 +159,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
-	github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d // indirect
+	github.com/benbjohnson/immutable v0.4.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.12.0 // indirect
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect

go.sum

@@ -177,6 +177,8 @@ github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
 github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d h1:2qVb9bsAMtmAfnxXltm+6eBzrrS7SZ52c3SedsulaMI=
 github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM=
+github.com/benbjohnson/immutable v0.4.3 h1:GYHcksoJ9K6HyAUpGxwZURrbTkXA0Dh4otXGqbhdrjA=
+github.com/benbjohnson/immutable v0.4.3/go.mod h1:qJIKKSmdqz1tVzNtst1DZzvaqOU1onk1rc03IeM3Owk=
 github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b h1:5JgaFtHFRnOPReItxvhMDXbvuBkjSWE+9glJyF466yw=
 github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b/go.mod h1:eMD2XUcPsHYbakFEocKrWZp47G0MRJYoC60qFblGjpA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

smt/pkg/db/mdbx.go

@@ -161,23 +161,23 @@ func (m *EriDb) SetDepth(depth uint8) error {
 	return m.tx.Put(TableStats, []byte(MetaDepth), []byte{depth})
 }
 
-func (m *EriRoDb) Get(key utils.NodeKey) (utils.NodeValue12, error) {
+func (m *EriRoDb) Get(key utils.NodeKey, values *utils.NodeValue12) error {
 	keyConc := utils.ArrayToScalar(key[:])
 	k := utils.ConvertBigIntToHex(keyConc)
 
 	data, err := m.kvTxRoSMT.GetOne(TableSmt, []byte(k))
 	if err != nil {
-		return utils.NodeValue12{}, err
+		return err
 	}
 
 	if data == nil || len(data) == 0 {
-		return utils.NodeValue12{}, nil
+		return nil
 	}
 
 	vConc := utils.ConvertHexToBigInt(string(data))
-	val := utils.ScalarToNodeValue(vConc)
+	utils.ScalarToNodeValue(vConc, (*[12]*big.Int)(values))
 
-	return val, nil
+	return nil
 }
 
 func (m *EriDb) Insert(key utils.NodeKey, value utils.NodeValue12) error {
@@ -347,7 +347,8 @@ func (m *EriRoDb) GetDb() map[string][]string {
 		hk := string(k)
 
 		vConc := utils.ConvertHexToBigInt(string(v))
-		val := utils.ScalarToNodeValue(vConc)
+		val := utils.NewNodeValue12()
+		utils.ScalarToNodeValue(vConc, (*[12]*big.Int)(&val))
 
 		truncationLength := 12