Feat/trie mutex refactor

common/interface.go

@@ -39,7 +39,7 @@ type BufferedErrChan interface {
 type Trie interface {
 	Get(key []byte) ([]byte, uint32, error)
 	Update(key, value []byte) error
-	Delete(key []byte) error
+	Delete(key []byte)
 	RootHash() ([]byte, error)
 	Commit() error
 	Recreate(root []byte) (Trie, error)
@@ -371,3 +371,26 @@ type ExecutionOrderGetter interface {
 	Len() int
 	IsInterfaceNil() bool
 }
+
+// TrieBatcher defines the methods needed for a trie batcher
+type TrieBatcher interface {
+	BatchHandler
+	GetSortedDataForInsertion() ([]string, map[string]core.TrieData)
+	GetSortedDataForRemoval() []string
+	IsInterfaceNil() bool
+}
+
+// TrieBatchManager defines the methods needed for managing the trie batch
+type TrieBatchManager interface {
+	BatchHandler
+	MarkTrieUpdateInProgress() (TrieBatcher, error)
+	MarkTrieUpdateCompleted()
+	IsInterfaceNil() bool
+}
+
+// BatchHandler is the interface for the batch handler
+type BatchHandler interface {
+	Add(key []byte, data core.TrieData)
+	MarkForRemoval(key []byte)
+	Get(key []byte) ([]byte, bool)
+}

integrationTests/state/genesisState/genesisState_test.go

@@ -364,7 +364,7 @@ func execute(
 	for _, idx := range randomRemovablePairsIdx {
 		tPair := totalPairs[idx]
 
-		_ = tr.Delete(tPair.key)
+		tr.Delete(tPair.key)
 	}
 	finalRootHash, _ := tr.RootHash()
 

state/accountsDB.go

@@ -340,11 +340,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error
 	}
 
 	if oldCodeEntry.NumReferences <= 1 {
-		err = adb.mainTrie.Delete(oldCodeHash)
-		if err != nil {
-			return nil, err
-		}
-
+		adb.mainTrie.Delete(oldCodeHash)
 		return unmodifiedOldCodeEntry, nil
 	}
 
@@ -528,7 +524,8 @@ func (adb *AccountsDB) RemoveAccount(address []byte) error {
 		"address", hex.EncodeToString(address),
 	)
 
-	return adb.mainTrie.Delete(address)
+	adb.mainTrie.Delete(address)
+	return nil
 }
 
 func (adb *AccountsDB) removeCodeAndDataTrie(acnt vmcommon.AccountHandler) error {

state/accountsDB_test.go

@@ -446,9 +446,8 @@ func TestAccountsDB_RemoveAccountShouldWork(t *testing.T) {
 			serializedAcc, err := marshaller.Marshal(stateMock.AccountWrapMock{})
 			return serializedAcc, 0, err
 		},
-		DeleteCalled: func(key []byte) error {
+		DeleteCalled: func(key []byte) {
 			wasCalled = true
-			return nil
 		},
 		GetStorageManagerCalled: func() common.StorageManager {
 			return &storageManager.StorageManagerStub{}

state/trackableDataTrie/trackableDataTrie.go

@@ -241,10 +241,7 @@ func (tdt *trackableDataTrie) updateTrie(dtr state.DataTrie) ([]core.TrieData, e
 		}
 		oldValues[index] = oldVal
 
-		err = tdt.deleteOldEntryIfMigrated([]byte(key), dataEntry, oldVal)
-		if err != nil {
-			return nil, err
-		}
+		tdt.deleteOldEntryIfMigrated([]byte(key), dataEntry, oldVal)
 
 		newKey, err := tdt.modifyTrie([]byte(key), dataEntry, oldVal, dtr)
 		if err != nil {
@@ -340,23 +337,22 @@ func (tdt *trackableDataTrie) getValueNotSpecifiedVersion(key []byte, val []byte
 	return trimmedValue, nil
 }
 
-func (tdt *trackableDataTrie) deleteOldEntryIfMigrated(key []byte, newData dirtyData, oldEntry core.TrieData) error {
+func (tdt *trackableDataTrie) deleteOldEntryIfMigrated(key []byte, newData dirtyData, oldEntry core.TrieData) {
 	if !tdt.enableEpochsHandler.IsFlagEnabled(common.AutoBalanceDataTriesFlag) {
-		return nil
+		return
 	}
 
 	isMigration := oldEntry.Version == core.NotSpecified && newData.newVersion == core.AutoBalanceEnabled
 	if isMigration && len(newData.value) != 0 {
 		log.Trace("delete old entry if migrated", "key", key)
-		return tdt.tr.Delete(key)
+		tdt.tr.Delete(key)
 	}
-
-	return nil
 }
 
 func (tdt *trackableDataTrie) modifyTrie(key []byte, dataEntry dirtyData, oldVal core.TrieData, dtr state.DataTrie) ([]byte, error) {
 	if len(dataEntry.value) == 0 {
-		return nil, tdt.deleteFromTrie(oldVal, key, dtr)
+		tdt.deleteFromTrie(oldVal, key, dtr)
+		return nil, nil
 	}
 
 	version := dataEntry.newVersion
@@ -369,20 +365,19 @@ func (tdt *trackableDataTrie) modifyTrie(key []byte, dataEntry dirtyData, oldVal
 	return newKey, dtr.UpdateWithVersion(newKey, value, version)
 }
 
-func (tdt *trackableDataTrie) deleteFromTrie(oldVal core.TrieData, key []byte, dtr state.DataTrie) error {
+func (tdt *trackableDataTrie) deleteFromTrie(oldVal core.TrieData, key []byte, dtr state.DataTrie) {
 	if len(oldVal.Value) == 0 {
-		return nil
+		return
 	}
 
 	if oldVal.Version == core.AutoBalanceEnabled {
-		return dtr.Delete(tdt.hasher.Compute(string(key)))
+		dtr.Delete(tdt.hasher.Compute(string(key)))
+		return
 	}
 
 	if oldVal.Version == core.NotSpecified {
-		return dtr.Delete(key)
+		dtr.Delete(key)
 	}
-
-	return nil
 }
 
 // IsInterfaceNil returns true if there is no value under the interface

state/trackableDataTrie/trackableDataTrie_test.go

@@ -424,10 +424,9 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 				updateCalled = true
 				return nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Equal(t, expectedKey, key)
 				deleteCalled = true
-				return nil
 			},
 		}
 
@@ -476,9 +475,8 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 				updateCalled = true
 				return nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Fail(t, "this should not have been called")
-				return nil
 			},
 		}
 
@@ -537,9 +535,8 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 				updateCalled = true
 				return nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Fail(t, "this delete should not have been called")
-				return nil
 			},
 		}
 
@@ -587,9 +584,8 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 				updateCalled = true
 				return nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Fail(t, "this delete should not have been called")
-				return nil
 			},
 		}
 
@@ -643,10 +639,9 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 			GetCalled: func(key []byte) ([]byte, uint32, error) {
 				return []byte("value"), 0, nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Equal(t, expectedKey, key)
 				updateCalled = true
-				return nil
 			},
 		}
 
@@ -669,10 +664,9 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 			GetCalled: func(key []byte) ([]byte, uint32, error) {
 				return nil, 0, nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Equal(t, expectedKey, key)
 				deleteCalled = true
-				return nil
 			},
 		}
 
@@ -700,10 +694,9 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 
 				return nil, 0, nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Equal(t, hasher.Compute(string(expectedKey)), key)
 				deleteCalled = true
-				return nil
 			},
 		}
 
@@ -735,10 +728,9 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 
 				return nil, 0, nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Equal(t, expectedKey, key)
 				deleteCalled++
-				return nil
 			},
 		}
 
@@ -779,9 +771,8 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) {
 				updateCalled = true
 				return nil
 			},
-			DeleteCalled: func(key []byte) error {
+			DeleteCalled: func(key []byte) {
 				assert.Fail(t, "this delete should not have been called")
-				return nil
 			},
 		}
 

testscommon/trie/trieStub.go

@@ -16,7 +16,7 @@ type TrieStub struct {
 	GetCalled                       func(key []byte) ([]byte, uint32, error)
 	UpdateCalled                    func(key, value []byte) error
 	UpdateWithVersionCalled         func(key, value []byte, version core.TrieNodeVersion) error
-	DeleteCalled                    func(key []byte) error
+	DeleteCalled                    func(key []byte)
 	RootCalled                      func() ([]byte, error)
 	CommitCalled                    func() error
 	RecreateCalled                  func(root []byte) (common.Trie, error)
@@ -109,12 +109,10 @@ func (ts *TrieStub) CollectLeavesForMigration(args vmcommon.ArgsMigrateDataTrieL
 }
 
 // Delete -
-func (ts *TrieStub) Delete(key []byte) error {
+func (ts *TrieStub) Delete(key []byte) {
 	if ts.DeleteCalled != nil {
-		return ts.DeleteCalled(key)
+		ts.DeleteCalled(key)
 	}
-
-	return errNotImplemented
 }
 
 // RootHash -

trie/baseIterator_test.go

@@ -35,10 +35,12 @@ func TestBaseIterator_HasNext(t *testing.T) {
 
 	tr := emptyTrie()
 	_ = tr.Update([]byte("dog"), []byte("dog"))
+	trie.ExecuteUpdatesFromBatch(tr)
 	it, _ := trie.NewBaseIterator(tr)
 	assert.False(t, it.HasNext())
 
 	_ = tr.Update([]byte("doe"), []byte("doe"))
+	trie.ExecuteUpdatesFromBatch(tr)
 	it, _ = trie.NewBaseIterator(tr)
 	assert.True(t, it.HasNext())
 }
@@ -78,6 +80,7 @@ func TestIterator_Search(t *testing.T) {
 	_ = tr.Update([]byte("dog"), []byte("puppy"))
 	_ = tr.Update([]byte("ddog"), []byte("cat"))
 	_ = tr.Update([]byte("ddoge"), []byte("foo"))
+	trie.ExecuteUpdatesFromBatch(tr)
 
 	expectedHashes := []string{
 		"ecc2304769996585131ad6276c1422265813a2b79d60392130c4baa19a9b4e06",

trie/branchNode_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
 	"github.com/multiversx/mx-chain-go/testscommon/marshallerMock"
 	"github.com/multiversx/mx-chain-go/trie/statistics"
+	"github.com/multiversx/mx-chain-go/trie/trieBatchManager"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -82,6 +83,7 @@ func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) {
 		maxTrieLevelInMemory: 5,
 		chanClose:            make(chan struct{}),
 		enableEpochsHandler:  &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		batchManager:         trieBatchManager.NewTrieBatchManager(),
 	}
 
 	return tr, trieStorage
@@ -92,6 +94,7 @@ func initTrie() *patriciaMerkleTrie {
 	_ = tr.Update([]byte("doe"), []byte("reindeer"))
 	_ = tr.Update([]byte("dog"), []byte("puppy"))
 	_ = tr.Update([]byte("ddog"), []byte("cat"))
+	ExecuteUpdatesFromBatch(tr)
 
 	return tr
 }
@@ -207,6 +210,9 @@ func TestBranchNode_setRootHash(t *testing.T) {
 		_ = tr2.Update(val, val)
 	}
 
+	ExecuteUpdatesFromBatch(tr1)
+	ExecuteUpdatesFromBatch(tr2)
+
 	err := tr1.root.setRootHash()
 	_ = tr2.root.setHash()
 	assert.Nil(t, err)
@@ -943,7 +949,7 @@ func TestReduceBranchNodeWithExtensionNodeChildShouldWork(t *testing.T) {
 	_ = tr.Update([]byte("dog"), []byte("dog"))
 	_ = tr.Update([]byte("doll"), []byte("doll"))
 	_ = tr.Update([]byte("wolf"), []byte("wolf"))
-	_ = tr.Delete([]byte("wolf"))
+	tr.Delete([]byte("wolf"))
 
 	expectedHash, _ := expectedTr.RootHash()
 	hash, _ := tr.RootHash()
@@ -962,7 +968,7 @@ func TestReduceBranchNodeWithBranchNodeChildShouldWork(t *testing.T) {
 	_ = tr.Update([]byte("doe"), []byte("reindeer"))
 	_ = tr.Update([]byte("dog"), []byte("puppy"))
 	_ = tr.Update([]byte("dogglesworth"), []byte("cat"))
-	_ = tr.Delete([]byte("doe"))
+	tr.Delete([]byte("doe"))
 
 	expectedHash, _ := expectedTr.RootHash()
 	hash, _ := tr.RootHash()
@@ -981,7 +987,7 @@ func TestReduceBranchNodeWithLeafNodeChildShouldWork(t *testing.T) {
 	_ = tr.Update([]byte("doe"), []byte("reindeer"))
 	_ = tr.Update([]byte("dog"), []byte("puppy"))
 	_ = tr.Update([]byte("dogglesworth"), []byte("cat"))
-	_ = tr.Delete([]byte("dog"))
+	tr.Delete([]byte("dog"))
 
 	expectedHash, _ := expectedTr.RootHash()
 	hash, _ := tr.RootHash()
@@ -1000,7 +1006,7 @@ func TestReduceBranchNodeWithLeafNodeValueShouldWork(t *testing.T) {
 	_ = tr.Update([]byte("doe"), []byte("reindeer"))
 	_ = tr.Update([]byte("dog"), []byte("puppy"))
 	_ = tr.Update([]byte("dogglesworth"), []byte("cat"))
-	_ = tr.Delete([]byte("dogglesworth"))
+	tr.Delete([]byte("dogglesworth"))
 
 	expectedHash, _ := expectedTr.RootHash()
 	hash, _ := tr.RootHash()
@@ -1100,7 +1106,8 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) {
 	_ = tr.Commit()
 
 	tr.root, _ = tr.root.getCollapsed()
-	_ = tr.Delete([]byte("zzz"))
+	tr.Delete([]byte("zzz"))
+	ExecuteUpdatesFromBatch(tr)
 
 	assert.True(t, tr.root.isDirty())
 	assert.True(t, tr.root.isCollapsed())

trie/doubleListSync_test.go

@@ -60,6 +60,7 @@ func addDataToTrie(numKeysValues int, tr common.Trie) {
 
 		_ = tr.Update(keyVal, keyVal)
 	}
+	ExecuteUpdatesFromBatch(tr)
 }
 
 func createRequesterResolver(completeTrie common.Trie, interceptedNodes storage.Cacher, exceptionHashes [][]byte) RequestHandler {

trie/export_test.go

@@ -111,3 +111,8 @@ func GetDefaultTrieStorageManagerParameters() NewTrieStorageManagerArgs {
 		StatsCollector: statistics.NewStateStatistics(),
 	}
 }
+
+func ExecuteUpdatesFromBatch(tr common.Trie) {
+	pmt, _ := tr.(*patriciaMerkleTrie)
+	_ = pmt.updateTrie()
+}