diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 1a49ddc42b1..7df6f631998 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -887,6 +887,9 @@ AccountsStatePruningEnabled = false PeerStatePruningEnabled = true StateStatisticsEnabled = false + MaxUserTrieSizeInMemory = 524288000 #500MB + MaxPeerTrieSizeInMemory = 104857600 #100MB + DataTriesSizeInMemory = 524288000 #500MB [TrieLeavesRetrieverConfig] Enabled = false diff --git a/common/common.go b/common/common.go index 0ee094ccab1..b5bec600b2b 100644 --- a/common/common.go +++ b/common/common.go @@ -28,6 +28,9 @@ const ( nonceIndex = 0 ) +// TenMbSize defines the size of 10 megabytes in bytes, used as a constant for memory limits or buffer sizes +const TenMbSize = uint64(10485760) + type executionResultHandler interface { GetMiniBlockHeadersHandlers() []data.MiniBlockHeaderHandler } diff --git a/common/interface.go b/common/interface.go index c49b9781449..6b564102261 100644 --- a/common/interface.go +++ b/common/interface.go @@ -152,7 +152,6 @@ type SnapshotDbHandler interface { // TriesHolder is used to store multiple tries type TriesHolder interface { Put([]byte, Trie) - Replace(key []byte, tr Trie) Get([]byte) Trie GetAll() []Trie Reset() diff --git a/config/config.go b/config/config.go index 70b74c60dd3..426f025475c 100644 --- a/config/config.go +++ b/config/config.go @@ -458,6 +458,9 @@ type StateTriesConfig struct { AccountsStatePruningEnabled bool PeerStatePruningEnabled bool StateStatisticsEnabled bool + MaxUserTrieSizeInMemory uint64 + MaxPeerTrieSizeInMemory uint64 + DataTriesSizeInMemory uint64 } // StateAccessesCollectorConfig will hold information about state accesses collector diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go index 003d20a37b6..ed8037542a2 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go @@ -14,7 +14,7 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process/factory" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cache" @@ -86,10 +86,10 @@ func createStoreForMeta() dataRetriever.StorageService { } func createTriesHolderForMeta() common.TriesHolder { - triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) - return triesHolder + triesContainer := triesHolder.NewTriesHolder() + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) + return triesContainer } // ------- NewResolversContainerFactory diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go index e2a26be3733..74f99260ec9 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go @@ -14,7 +14,7 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process/factory" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cache" @@ -92,10 +92,10 @@ func createStoreForShard() dataRetriever.StorageService { } func createTriesHolderForShard() common.TriesHolder { - triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) - return triesHolder + triesContainer := triesHolder.NewTriesHolder() + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) + return triesContainer } // ------- NewResolversContainerFactory diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 39f8d35e291..70aa5ccf873 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -15,10 +15,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" + "github.com/multiversx/mx-chain-go/state/triesHolder" logger "github.com/multiversx/mx-chain-logger-go" - "github.com/multiversx/mx-chain-go/process/interceptors/processor" - "github.com/multiversx/mx-chain-go/common" disabledCommon "github.com/multiversx/mx-chain-go/common/disabled" "github.com/multiversx/mx-chain-go/common/ordering" @@ -42,11 +41,11 @@ import ( "github.com/multiversx/mx-chain-go/process/heartbeat/validator" "github.com/multiversx/mx-chain-go/process/interceptors" disabledInterceptors "github.com/multiversx/mx-chain-go/process/interceptors/disabled" + "github.com/multiversx/mx-chain-go/process/interceptors/processor" "github.com/multiversx/mx-chain-go/process/peer" "github.com/multiversx/mx-chain-go/redundancy" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" @@ -283,7 +282,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap, return nil, err } - epochStartProvider.trieContainer = state.NewDataTriesHolder() + epochStartProvider.trieContainer = triesHolder.NewTriesHolder() epochStartProvider.trieStorageManagers = make(map[string]common.StorageManager) if epochStartProvider.generalConfig.Hardfork.AfterHardFork { diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index a964dfefb77..7cabfa4e0bd 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -182,6 +182,9 @@ func createMockEpochStartBootstrapArgs( AccountsStatePruningEnabled: true, SnapshotsEnabled: true, PeerStatePruningEnabled: true, + MaxUserTrieSizeInMemory: generalCfg.StateTriesConfig.MaxUserTrieSizeInMemory, + MaxPeerTrieSizeInMemory: generalCfg.StateTriesConfig.MaxPeerTrieSizeInMemory, + DataTriesSizeInMemory: generalCfg.StateTriesConfig.DataTriesSizeInMemory, }, TrieStorageManagerConfig: config.TrieStorageManagerConfig{ PruningBufferLen: 1000, diff --git a/epochStart/metachain/baseRewards_test.go b/epochStart/metachain/baseRewards_test.go index b0e1118b84e..61eb184c5b9 100644 --- a/epochStart/metachain/baseRewards_test.go +++ b/epochStart/metachain/baseRewards_test.go @@ -29,7 +29,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -1140,11 +1139,11 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs { hasher := sha256.NewSha256() marshalizer := &marshal.GogoProtoMarshalizer{} - storageManagerArgs := storage.GetStorageManagerArgs() + storageManagerArgs := txExecOrderStub.GetStorageManagerArgs() storageManagerArgs.Marshalizer = marshalizer storageManagerArgs.Hasher = hasher - trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions()) + trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, txExecOrderStub.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ Hasher: hasher, Marshaller: marshalizer, diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index 1402780fc32..1aff5eb3e0e 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -45,6 +45,7 @@ import ( storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -748,7 +749,8 @@ func createAccountsDB( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) *state.AccountsDB { - tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler) + tenMbSize := uint64(10485760) + tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, tenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -757,14 +759,15 @@ func createAccountsDB( spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMbSize, } adb, _ := state.NewAccountsDB(args) return adb @@ -773,12 +776,12 @@ func createAccountsDB( func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEpochs, trieStorer storage.Storer) (ArgsNewEpochStartSystemSCProcessing, vm.SystemSCContainer) { hasher := sha256.NewSha256() marshalizer := &marshal.GogoProtoMarshalizer{} - storageManagerArgs := storageMock.GetStorageManagerArgs() + storageManagerArgs := testCommon.GetStorageManagerArgs() storageManagerArgs.Marshalizer = marshalizer storageManagerArgs.Hasher = hasher storageManagerArgs.MainStorer = trieStorer - trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions()) + trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, testCommon.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ Hasher: hasher, Marshaller: marshalizer, diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index 643ab09421c..9ef03e5bef1 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -594,6 +594,7 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(), StatsCollector: args.statusCoreComponents.StateStatsHandler(), + MaxSizeInMemory: args.generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory, } trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs) if err != nil { @@ -601,14 +602,16 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai } argsAPIAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: args.coreComponents.Hasher(), - Marshaller: args.coreComponents.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: args.coreComponents.AddressPubKeyConverter(), - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: merkleTrie, + Hasher: args.coreComponents.Hasher(), + Marshaller: args.coreComponents.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: args.coreComponents.AddressPubKeyConverter(), + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + // TODO check if this should be lower than in the processing adb + MaxDataTriesSizeInMemory: args.generalConfig.StateTriesConfig.DataTriesSizeInMemory, } provider, err := blockInfoProviders.NewCurrentBlockInfo(chainHandler) diff --git a/factory/api/apiResolverFactory_test.go b/factory/api/apiResolverFactory_test.go index 7c724ab2679..dd2af1535cc 100644 --- a/factory/api/apiResolverFactory_test.go +++ b/factory/api/apiResolverFactory_test.go @@ -37,7 +37,10 @@ import ( "github.com/stretchr/testify/require" ) -const unreachableStep = 10000 +const ( + unreachableStep = 10000 + tenMBSize = uint64(10485760) +) type failingSteps struct { marshallerStepCounter int @@ -305,7 +308,11 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs { TrieStorageManagerConfig: config.TrieStorageManagerConfig{ SnapshotsGoroutineNum: 1, }, - StateTriesConfig: config.StateTriesConfig{}, + StateTriesConfig: config.StateTriesConfig{ + MaxUserTrieSizeInMemory: tenMBSize, + MaxPeerTrieSizeInMemory: tenMBSize, + DataTriesSizeInMemory: tenMBSize, + }, VirtualMachine: config.VirtualMachineServicesConfig{ Querying: config.QueryVirtualMachineConfig{ VirtualMachineConfig: config.VirtualMachineConfig{ diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index cb4c8a71181..aa139f9ae51 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + common2 "github.com/multiversx/mx-chain-go/testscommon/common" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/stretchr/testify/require" @@ -26,7 +27,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/processMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - storageManager "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" ) @@ -91,13 +91,13 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { cryptoComponents := componentsMock.GetCryptoComponents(coreComponents) networkComponents := componentsMock.GetNetworkComponents(cryptoComponents) - storageManagerArgs := storageManager.GetStorageManagerArgs() + storageManagerArgs := common2.GetStorageManagerArgs() storageManagerArgs.Marshalizer = coreComponents.InternalMarshalizer() storageManagerArgs.Hasher = coreComponents.Hasher() - storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions()) + storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions()) storageManagerArgs.MainStorer = mock.NewMemDbMock() - storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions()) + storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions()) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser @@ -209,20 +209,22 @@ func createAccountAdapter( trieStorage common.StorageManager, handler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler) + tenMbSize := uint64(10485760) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, tenMbSize) if err != nil { return nil, err } args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMbSize, } adb, err := state.NewAccountsDB(args) if err != nil { diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index 0a2e4f278df..29aa97f51e4 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -253,15 +253,17 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. } argsProcessingAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotsManager, - StateAccessesCollector: StateAccessesCollector, - PruningEnabled: scf.config.StateTriesConfig.AccountsStatePruningEnabled, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotsManager, + StateAccessesCollector: StateAccessesCollector, + PruningEnabled: scf.config.StateTriesConfig.AccountsStatePruningEnabled, + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, + } accountsAdapter, err := state.NewAccountsDB(argsProcessingAccountsDB) if err != nil { @@ -280,14 +282,16 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. } argsAPIAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactoryAPI, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: disabled.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactoryAPI, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: disabled.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + // TODO check if this should be lower than in the processing adb + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, } accountsAdapterApiOnFinal, err := factoryState.CreateAccountsAdapterAPIOnFinal(argsAPIAccountsDB, scf.chainHandler) @@ -357,15 +361,17 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries // TODO: also collect state accesses for the peer trie argsProcessingPeerAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), - PruningEnabled: scf.config.StateTriesConfig.PeerStatePruningEnabled, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + PruningEnabled: scf.config.StateTriesConfig.PeerStatePruningEnabled, + // TODO check if this should be different from the user adb + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, } peerAdapter, err := state.NewPeerAccountsDB(argsProcessingPeerAccountsDB) if err != nil { diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index d5dada9caec..5c523d1d50a 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -57,8 +57,8 @@ func createMockArgument( entireSupply *big.Int, ) ArgsGenesisBlockCreator { - storageManagerArgs := storageCommon.GetStorageManagerArgs() - storageManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageCommon.GetStorageManagerOptions()) + storageManagerArgs := commonMocks.GetStorageManagerArgs() + storageManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, commonMocks.GetStorageManagerOptions()) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManager diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index a34ca035468..d2bf3e3bb50 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -19,20 +19,21 @@ func createAccountAdapter( addressConverter core.PubkeyConverter, enableEpochsHandler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, common.TenMbSize) if err != nil { return nil, err } args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: common.TenMbSize, } adb, err := state.NewAccountsDB(args) diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index 4d0bbf72ae0..668fd5256d2 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -15,8 +15,8 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" + testStorage "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - testStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/require" ) @@ -101,7 +101,7 @@ func generateTriesWithMaxDepth( ) []*keyForTrie { tries := make([]*keyForTrie, numTries) for i := 0; i < numTries; i++ { - tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) key := insertKeysIntoTrie(t, tr, numTrieLevels, numChildrenPerBranch) rootHash, _ := tr.RootHash() diff --git a/integrationTests/longTests/storage/storage_test.go b/integrationTests/longTests/storage/storage_test.go index 779f7587799..b6a68d5bd19 100644 --- a/integrationTests/longTests/storage/storage_test.go +++ b/integrationTests/longTests/storage/storage_test.go @@ -9,8 +9,8 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/integrationTests" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" ) @@ -106,17 +106,17 @@ func TestWriteContinuouslyInTree(t *testing.T) { nbTxsWrite := 1000000 testStorage := integrationTests.NewTestStorage() store := testStorage.CreateStorageLevelDB() - storageManagerArgs := storage.GetStorageManagerArgs() + storageManagerArgs := testCommon.GetStorageManagerArgs() storageManagerArgs.MainStorer = store storageManagerArgs.Marshalizer = &marshal.JsonMarshalizer{} storageManagerArgs.Hasher = blake2b.NewBlake2b() - options := storage.GetStorageManagerOptions() + options := testCommon.GetStorageManagerOptions() options.PruningEnabled = false trieStorage, _ := trie.CreateTrieStorageManager(storageManagerArgs, options) - tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) defer func() { _ = store.DestroyUnit() diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 08efa98b699..005125a2ea4 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -25,6 +25,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/sha256" crypto "github.com/multiversx/mx-chain-crypto-go" "github.com/multiversx/mx-chain-go/epochStart/notifier" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-storage-go/types" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -54,7 +55,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - testStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -271,12 +271,12 @@ func TestAccountsDB_CommitTwoOkAccountsShouldWork(t *testing.T) { func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { hasher := integrationTests.TestHasher store := integrationTests.CreateMemUnit() - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = store args.Hasher = hasher trieStorage, _ := trie.NewTrieStorageManager(args) - tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) key := hasher.Compute("key") value := hasher.Compute("value") @@ -1055,10 +1055,10 @@ func createAccounts( HashesSize: evictionWaitListSize * 100, } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = store trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, @@ -1079,14 +1079,15 @@ func createAccounts( StateStatsHandler: statistics.NewStateStatistics(), }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2731,10 +2732,10 @@ func createAccountsDBTestSetup() *state.AccountsDB { HashesSize: evictionWaitListSize * 100, } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.GeneralConfig = generalCfg trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, @@ -2757,14 +2758,15 @@ func createAccountsDBTestSetup() *state.AccountsDB { }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2786,10 +2788,10 @@ func TestStateSnapshot_MultipleEpochsWithoutCompleteSnapshot(t *testing.T) { numEpochs := 5 getCounters := make([]int, numEpochs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = mainStorer trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) defer func() { _ = trieStorage.Close() }() diff --git a/integrationTests/state/stateTrieClose/stateTrieClose_test.go b/integrationTests/state/stateTrieClose/stateTrieClose_test.go index 699a4b085f2..8a638c8a32a 100644 --- a/integrationTests/state/stateTrieClose/stateTrieClose_test.go +++ b/integrationTests/state/stateTrieClose/stateTrieClose_test.go @@ -12,9 +12,9 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/state/parsers" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/stretchr/testify/assert" @@ -23,7 +23,7 @@ import ( func TestPatriciaMerkleTrie_Close(t *testing.T) { numLeavesToAdd := 200 trieStorage, _ := integrationTests.CreateTrieStorageManager(integrationTests.CreateMemUnit()) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) for i := 0; i < numLeavesToAdd; i++ { _ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) @@ -139,7 +139,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { } func TestTrieStorageManager_Close(t *testing.T) { - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() gc := goroutines.NewGoCounter(goroutines.TestsRelevantGoRoutines) idxInitial, _ := gc.Snapshot() diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 8a3ca8603bf..cd19b7dff18 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -81,7 +81,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/stakingcommon" testStorage "github.com/multiversx/mx-chain-go/testscommon/state" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" - testcommonStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/txDataBuilder" "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/vm" @@ -104,6 +103,9 @@ var InitialRating = uint32(50) // AdditionalGasLimit is the value that can be added on a transaction in the GasLimit var AdditionalGasLimit = uint64(999000) +// TenMbSize represents 10 MB in bytes +const TenMbSize = uint64(10485760) + // GasSchedulePath -- const GasSchedulePath = "../../../../cmd/node/config/gasSchedules/gasScheduleV4.toml" @@ -430,7 +432,7 @@ func CreateTrieStorageManagerWithPruningStorer(coordinator sharding.Coordinator, fmt.Println("err creating main storer" + err.Error()) } - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.MainStorer = mainStorer args.Marshalizer = TestMarshalizer args.Hasher = TestHasher @@ -442,7 +444,7 @@ func CreateTrieStorageManagerWithPruningStorer(coordinator sharding.Coordinator, // CreateTrieStorageManager creates the trie storage manager for the tests func CreateTrieStorageManager(store storage.Storer) (common.StorageManager, storage.Storer) { - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.MainStorer = store args.Marshalizer = TestMarshalizer args.Hasher = TestHasher @@ -466,7 +468,7 @@ func CreateAccountsDBWithEnableEpochsHandler( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) (*state.AccountsDB, common.Trie) { - tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler) + tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler, TenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, @@ -490,15 +492,16 @@ func CreateAccountsDBWithEnableEpochsHandler( _ = snapshotsManager.SetSyncer(&mock.AccountsDBSyncerStub{}) args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: sha256.NewSha256(), - Marshaller: TestMarshalizer, - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), - PruningEnabled: trieStorageManager.IsPruningEnabled(), + Trie: tr, + Hasher: sha256.NewSha256(), + Marshaller: TestMarshalizer, + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + PruningEnabled: trieStorageManager.IsPruningEnabled(), + MaxDataTriesSizeInMemory: TenMbSize, } adb, _ := state.NewAccountsDB(args) @@ -1157,13 +1160,13 @@ func CreateSimpleTxProcessor(accnts state.AccountsAdapter) process.TransactionPr // CreateNewDefaultTrie returns a new trie with test hasher and marsahalizer func CreateNewDefaultTrie() common.Trie { - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.Marshalizer = TestMarshalizer args.Hasher = TestHasher trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, TenMbSize) return tr } diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 0a4f06b6d14..8b510a6e9d6 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -31,6 +31,7 @@ import ( ed25519SingleSig "github.com/multiversx/mx-chain-crypto-go/signing/ed25519/singlesig" "github.com/multiversx/mx-chain-crypto-go/signing/mcl" mclsig "github.com/multiversx/mx-chain-crypto-go/signing/mcl/singlesig" + "github.com/multiversx/mx-chain-go/state/triesHolder" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-vm-common-go/parsers" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" @@ -705,7 +706,7 @@ func (tpn *TestProcessorNode) initAccountDBsWithPruningStorer() { tpn.EpochStartNotifier = notifier.NewEpochStartSubscriptionHandler() } trieStorageManager := CreateTrieStorageManagerWithPruningStorer(tpn.ShardCoordinator, tpn.EpochStartNotifier) - tpn.TrieContainer = state.NewDataTriesHolder() + tpn.TrieContainer = triesHolder.NewTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) tpn.AccntStateProposal, _ = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) @@ -722,7 +723,7 @@ func (tpn *TestProcessorNode) initAccountDBsWithPruningStorer() { func (tpn *TestProcessorNode) initAccountDBs(store storage.Storer) { trieStorageManager, _ := CreateTrieStorageManager(store) - tpn.TrieContainer = state.NewDataTriesHolder() + tpn.TrieContainer = triesHolder.NewTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) tpn.AccntStateProposal, _ = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) diff --git a/integrationTests/vm/staking/componentsHolderCreator.go b/integrationTests/vm/staking/componentsHolderCreator.go index f2965b90780..226556b1b96 100644 --- a/integrationTests/vm/staking/componentsHolderCreator.go +++ b/integrationTests/vm/staking/componentsHolderCreator.go @@ -209,6 +209,7 @@ func createAccountsDB( coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), + integrationTests.TenMbSize, ) argsEvictionWaitingList := evictionWaitingList.MemoryEvictionWaitingListArgs{ @@ -218,14 +219,15 @@ func createAccountsDB( ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(argsEvictionWaitingList) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccountsDb := state.ArgsAccountsDB{ - Trie: tr, - Hasher: coreComponents.Hasher(), - Marshaller: coreComponents.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: coreComponents.AddressPubKeyConverter(), - SnapshotsManager: &stateTests.SnapshotsManagerStub{}, - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: coreComponents.Hasher(), + Marshaller: coreComponents.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: coreComponents.AddressPubKeyConverter(), + SnapshotsManager: &stateTests.SnapshotsManagerStub{}, + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDb) return adb diff --git a/process/sync/baseForkDetector_test.go b/process/sync/baseForkDetector_test.go index 2fa476de1df..c140471b48f 100644 --- a/process/sync/baseForkDetector_test.go +++ b/process/sync/baseForkDetector_test.go @@ -5,9 +5,6 @@ import ( "testing" "time" - "github.com/multiversx/mx-chain-go/testscommon/chainParameters" - "github.com/multiversx/mx-chain-go/testscommon/processMocks" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" @@ -17,8 +14,10 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/sync" "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/chainParameters" "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" + "github.com/multiversx/mx-chain-go/testscommon/processMocks" "github.com/stretchr/testify/assert" ) diff --git a/state/accountsDB.go b/state/accountsDB.go index b412bac97d6..093005a9bf8 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -23,6 +23,7 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/state/parsers" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" ) @@ -98,15 +99,16 @@ var log = logger.GetOrCreate("state") // ArgsAccountsDB is the arguments DTO for the AccountsDB instance type ArgsAccountsDB struct { - Trie common.Trie - Hasher hashing.Hasher - Marshaller marshal.Marshalizer - AccountFactory AccountFactory - StoragePruningManager StoragePruningManager - AddressConverter core.PubkeyConverter - SnapshotsManager SnapshotsManager - StateAccessesCollector StateAccessesCollector - PruningEnabled bool + Trie common.Trie + Hasher hashing.Hasher + Marshaller marshal.Marshalizer + AccountFactory AccountFactory + StoragePruningManager StoragePruningManager + AddressConverter core.PubkeyConverter + SnapshotsManager SnapshotsManager + StateAccessesCollector StateAccessesCollector + PruningEnabled bool + MaxDataTriesSizeInMemory uint64 } // NewAccountsDB creates a new account manager @@ -116,10 +118,15 @@ func NewAccountsDB(args ArgsAccountsDB) (*AccountsDB, error) { return nil, err } - return createAccountsDb(args), nil + return createAccountsDb(args) } -func createAccountsDb(args ArgsAccountsDB) *AccountsDB { +func createAccountsDb(args ArgsAccountsDB) (*AccountsDB, error) { + dth, err := triesHolder.NewDataTriesHolder(args.MaxDataTriesSizeInMemory) + if err != nil { + return nil, fmt.Errorf("cannot create data tries holder: %w", err) + } + return &AccountsDB{ mainTrie: args.Trie, hasher: args.Hasher, @@ -128,7 +135,7 @@ func createAccountsDb(args ArgsAccountsDB) *AccountsDB { storagePruningManager: args.StoragePruningManager, entries: make([]JournalEntry, 0), mutOp: sync.RWMutex{}, - dataTries: NewDataTriesHolder(), + dataTries: dth, obsoleteDataTrieHashes: make(map[string][][]byte), loadCodeMeasurements: &loadingMeasurements{ identifier: "load code", @@ -137,7 +144,7 @@ func createAccountsDb(args ArgsAccountsDB) *AccountsDB { snapshotsManger: args.SnapshotsManager, stateAccessesCollector: args.StateAccessesCollector, pruningEnabled: args.PruningEnabled, - } + }, nil } func checkArgsAccountsDB(args ArgsAccountsDB) error { diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index c296aed6037..103952a18b3 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -41,17 +41,20 @@ import ( "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/storageManager" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" ) -const trieDbOperationDelay = time.Second +const ( + trieDbOperationDelay = time.Second + tenMbSize = uint64(10485760) +) func createMockAccountsDBArgs() state.ArgsAccountsDB { accCreator := &stateMock.AccountsFactoryStub{ @@ -78,13 +81,14 @@ func createMockAccountsDBArgs() state.ArgsAccountsDB { return &storageManager.StorageManagerStub{} }, }, - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - AccountFactory: accCreator, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + AccountFactory: accCreator, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMbSize, } } @@ -141,10 +145,10 @@ func getDefaultStateComponents( marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = db trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, tenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -173,15 +177,16 @@ func getDefaultStateComponents( collector, _ := stateAccesses.NewCollector(stateDisabled.NewDisabledStateAccessesStorer(), stateAccesses.WithCollectWrite()) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: collector, - PruningEnabled: true, + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: collector, + PruningEnabled: true, + MaxDataTriesSizeInMemory: tenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2126,9 +2131,9 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} ewl := stateMock.NewEvictionWaitingListMock(100) - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) argsAccountsDB := createMockAccountsDBArgs() @@ -2208,9 +2213,9 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { hasher := &hashingMocks.HasherMock{} ewl := stateMock.NewEvictionWaitingListMock(100) - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) argsAccountsDB := createMockAccountsDBArgs() @@ -2412,9 +2417,9 @@ func TestAccountsDB_GetCode(t *testing.T) { marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.Trie = tr @@ -2867,9 +2872,9 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() @@ -3192,8 +3197,8 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} enableEpochsHandler := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() - tsm, _ := trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochsHandler) + tsm, _ := trie.NewTrieStorageManager(testCommon.GetStorageManagerArgs()) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochsHandler, tenMbSize) spm := &stateMock.StoragePruningManagerStub{} argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.PruningEnabled = true diff --git a/state/dataTriesHolder_test.go b/state/dataTriesHolder_test.go deleted file mode 100644 index 8e65f1bb3b2..00000000000 --- a/state/dataTriesHolder_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package state_test - -import ( - "strconv" - "sync" - "testing" - - "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-go/state" - trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" - "github.com/stretchr/testify/assert" -) - -func TestNewDataTriesHolder(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - assert.False(t, check.IfNil(dth)) -} - -func TestDataTriesHolder_PutAndGet(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - tr := dth.Get([]byte("trie1")) - - assert.True(t, tr == tr1) -} - -func TestDataTriesHolder_Replace(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - tr2 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Replace([]byte("trie1"), tr2) - retrievedTrie := dth.Get([]byte("trie1")) - - assert.True(t, retrievedTrie == tr2) - assert.True(t, retrievedTrie != tr1) -} - -func TestDataTriesHolder_GetAll(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - tr2 := &trieMock.TrieStub{} - tr3 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Put([]byte("trie2"), tr2) - dth.Put([]byte("trie3"), tr3) - tries := dth.GetAll() - - assert.Equal(t, 3, len(tries)) -} - -func TestDataTriesHolder_Reset(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Reset() - - tr := dth.Get([]byte("trie1")) - assert.Nil(t, tr) -} - -func TestDataTriesHolder_Concurrency(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - numTries := 50 - - wg := sync.WaitGroup{} - wg.Add(numTries) - - for i := 0; i < numTries; i++ { - go func(key int) { - dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) - wg.Done() - }(i) - } - - wg.Wait() - - tries := dth.GetAll() - assert.Equal(t, numTries, len(tries)) -} - -func TestDataTriesHolder_GetAllTries(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - numTries := 50 - - wg := sync.WaitGroup{} - wg.Add(numTries) - - for i := 0; i < numTries; i++ { - go func(key int) { - dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) - wg.Done() - }(i) - } - - wg.Wait() - - tries := dth.GetAllTries() - assert.Equal(t, numTries, len(tries)) -} diff --git a/state/factory/accountsAdapterAPICreator_test.go b/state/factory/accountsAdapterAPICreator_test.go index a735fc7f9ce..0fd272719cb 100644 --- a/state/factory/accountsAdapterAPICreator_test.go +++ b/state/factory/accountsAdapterAPICreator_test.go @@ -17,19 +17,21 @@ import ( ) func createMockAccountsArgs() state.ArgsAccountsDB { + tenMbSize := uint64(10 * 1024 * 1024) return state.ArgsAccountsDB{ Trie: &mockTrie.TrieStub{ GetStorageManagerCalled: func() common.StorageManager { return &storageManager.StorageManagerStub{} }, }, - Hasher: &testscommon.HasherStub{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - AccountFactory: &mockState.AccountsFactoryStub{}, - StoragePruningManager: &mockState.StoragePruningManagerStub{}, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: &mockState.SnapshotsManagerStub{}, - StateAccessesCollector: &mockState.StateAccessesCollectorStub{}, + Hasher: &testscommon.HasherStub{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + AccountFactory: &mockState.AccountsFactoryStub{}, + StoragePruningManager: &mockState.StoragePruningManagerStub{}, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: &mockState.SnapshotsManagerStub{}, + StateAccessesCollector: &mockState.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMbSize, } } diff --git a/state/peerAccountsDB.go b/state/peerAccountsDB.go index 093e6d3b6e2..8545bdb9ff7 100644 --- a/state/peerAccountsDB.go +++ b/state/peerAccountsDB.go @@ -17,8 +17,13 @@ func NewPeerAccountsDB(args ArgsAccountsDB) (*PeerAccountsDB, error) { return nil, err } + originalAccountsDb, err := createAccountsDb(args) + if err != nil { + return nil, err + } + adb := &PeerAccountsDB{ - AccountsDB: createAccountsDb(args), + AccountsDB: originalAccountsDb, } return adb, nil diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index e188bd93058..cc06ec8024b 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -15,11 +15,11 @@ import ( "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -31,9 +31,10 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. } marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -62,14 +63,15 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMBSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/state/syncer/baseAccountsSyncer.go b/state/syncer/baseAccountsSyncer.go index 4064279a83f..ee84ac1f352 100644 --- a/state/syncer/baseAccountsSyncer.go +++ b/state/syncer/baseAccountsSyncer.go @@ -215,7 +215,8 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { b.mutex.Lock() defer b.mutex.Unlock() - dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.enableEpochsHandler) + tenMBSize := uint64(10485760) + dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.enableEpochsHandler, tenMBSize) if err != nil { log.Warn("error creating a new trie in baseAccountsSyncer.GetSyncedTries", "error", err) return make(map[string]common.Trie) diff --git a/state/syncer/userAccountSyncer_test.go b/state/syncer/userAccountSyncer_test.go index 4d7ac733baa..6bbd82b6847 100644 --- a/state/syncer/userAccountSyncer_test.go +++ b/state/syncer/userAccountSyncer_test.go @@ -89,7 +89,8 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/state/syncer/userAccountsSyncer_test.go b/state/syncer/userAccountsSyncer_test.go index d8e8685a114..3a7e108b3f0 100644 --- a/state/syncer/userAccountsSyncer_test.go +++ b/state/syncer/userAccountsSyncer_test.go @@ -32,6 +32,8 @@ import ( "github.com/multiversx/mx-chain-go/trie/storageMarker" ) +const tenMBSize = uint64(10485760) + func getDefaultUserAccountsSyncerArgs() syncer.ArgsNewUserAccountsSyncer { return syncer.ArgsNewUserAccountsSyncer{ ArgsNewBaseAccountsSyncer: getDefaultBaseAccSyncerArgs(), @@ -111,7 +113,7 @@ func getSerializedTrieNode( }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) _ = tr.Update(key, []byte("value")) _ = tr.Commit() @@ -162,7 +164,7 @@ func TestUserAccountsSyncer_SyncAccounts(t *testing.T) { }) } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler) { +func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint64) { marshalizer := &testscommon.ProtobufMarshalizerMock{} hasher := &testscommon.KeccakMock{} @@ -184,7 +186,7 @@ func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, has trieStorageManager, _ := trie.NewTrieStorageManager(args) - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{} + return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize } func emptyTrie() common.Trie { @@ -236,7 +238,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -293,7 +295,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -360,7 +362,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/state/triesHolder/dataTriesHolder.go b/state/triesHolder/dataTriesHolder.go new file mode 100644 index 00000000000..cc1cba71807 --- /dev/null +++ b/state/triesHolder/dataTriesHolder.go @@ -0,0 +1,213 @@ +package triesHolder + +import ( + "fmt" + "math" + "sync" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/storage" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-storage-go/lrucache/capacity" +) + +const maxTrieSizeMinValue = 1 * 1024 * 1024 // 1 MB + +var log = logger.GetOrCreate("state/dataTriesHolder") + +// dataTriesHolder is a structure that holds a map of tries and manages their memory usage +// It uses a doubly linked list to keep track of the order in which the tries were used +// and evicts the oldest used tries when the total size exceeds a maximum limit. +type dataTriesHolder struct { + cacher storage.AdaptedSizedLRUCache + dirtyTries map[string]struct{} // These are the tries that have been modified and need to be persisted + touchedTries map[string]struct{} // These are needed to compute an accurate totalTriesSize + evictedBuffer map[string]common.Trie // in case eviction happens for a dirty trie, we keep it here until we commit it + + mutex sync.RWMutex +} + +// NewDataTriesHolder creates a new instance of dataTriesHolder +func NewDataTriesHolder(maxTriesSize uint64) (*dataTriesHolder, error) { + if maxTriesSize < maxTrieSizeMinValue { + return nil, fmt.Errorf("%w, provided %d, minimum %d", ErrInvalidMaxTrieSizeValue, maxTriesSize, maxTrieSizeMinValue) + } + log.Trace("creating new data tries holder", "maxTriesSize", maxTriesSize) + + c, err := capacity.NewCapacityLRU(math.MaxInt, int64(maxTriesSize)) + if err != nil { + return nil, err + } + + return &dataTriesHolder{ + cacher: c, + dirtyTries: make(map[string]struct{}), + touchedTries: make(map[string]struct{}), + evictedBuffer: make(map[string]common.Trie), + mutex: sync.RWMutex{}, + }, nil +} + +// Put adds a trie pointer to the tries map +func (dth *dataTriesHolder) Put(key []byte, tr common.Trie) { + dth.mutex.Lock() + defer dth.mutex.Unlock() + + dth.putNoLock(key, tr) +} + +func (dth *dataTriesHolder) putNoLock(key []byte, tr common.Trie) { + if check.IfNil(tr) || len(key) == 0 { + log.Warn("trying to put nil trie or empty key in dataTriesHolder", "key", key, "trie", tr) + return + } + + keyString := string(key) + + if len(dth.evictedBuffer) > 0 { + // this means that this trie was evicted while being dirty + delete(dth.evictedBuffer, keyString) + } + + evicted := dth.cacher.AddSizedAndReturnEvicted(keyString, tr, int64(tr.SizeInMemory())) + dth.dirtyTries[keyString] = struct{}{} + dth.touchedTries[keyString] = struct{}{} + + if len(evicted) == 0 { + return + } + + for evictedKey, evictedValue := range evicted { + evictedKeyString, ok := evictedKey.(string) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", evictedKey)) + continue + } + _, ok = dth.dirtyTries[evictedKeyString] + if !ok { + continue + } + + evictedTrie, ok := evictedValue.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", evictedValue)) + continue + } + dth.evictedBuffer[evictedKeyString] = evictedTrie + } +} + +// Get returns the trie pointer that is stored in the map at the given key +func (dth *dataTriesHolder) Get(key []byte) common.Trie { + if len(key) == 0 { + return nil + } + + dth.mutex.Lock() + defer dth.mutex.Unlock() + + keyString := string(key) + + val, ok := dth.cacher.Get(keyString) + if !ok { + // maybe it was evicted while being dirty + evictedTr, exists := dth.evictedBuffer[keyString] + if !exists { + return nil + } + + delete(dth.evictedBuffer, keyString) + dth.putNoLock(key, evictedTr) + return evictedTr + } + + dth.touchedTries[keyString] = struct{}{} + tr, ok := val.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", val)) + return nil + } + + return tr +} + +// GetAll returns all the tries that are marked as dirty for this implementation. +// It also resets their dirty flag and recomputes the total size. +func (dth *dataTriesHolder) GetAll() []common.Trie { + dth.mutex.Lock() + defer dth.mutex.Unlock() + + tries := make([]common.Trie, 0) + for keyString := range dth.dirtyTries { + tr := dth.getDirtyTrieNoLock(keyString) + if check.IfNil(tr) { + continue + } + tries = append(tries, tr) + } + dth.dirtyTries = make(map[string]struct{}) + dth.evictedBuffer = make(map[string]common.Trie) + dth.recomputeTotalSize() + return tries +} + +func (dth *dataTriesHolder) getDirtyTrieNoLock(key string) common.Trie { + entry, exists := dth.cacher.Get(key) + if exists { + tr, ok := entry.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", entry)) + return nil + } + + return tr + } + + tr, ok := dth.evictedBuffer[key] + if !ok { + return nil + } + return tr +} + +func (dth *dataTriesHolder) recomputeTotalSize() { + for keyString := range dth.touchedTries { + entry, exists := dth.cacher.Get(keyString) + if !exists { + continue + } + tr, ok := entry.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", entry)) + continue + } + + evicted := dth.cacher.AddSized(keyString, tr, int64(tr.SizeInMemory())) + if evicted { + log.Warn("unexpected eviction while recomputing total size in dataTriesHolder") + } + } + dth.touchedTries = make(map[string]struct{}) +} + +// Reset clears the tries map +func (dth *dataTriesHolder) Reset() { + dth.mutex.Lock() + dth.reset() + dth.mutex.Unlock() +} + +func (dth *dataTriesHolder) reset() { + log.Trace("reset data tries holder") + + dth.cacher.Purge() + dth.dirtyTries = make(map[string]struct{}) + dth.touchedTries = make(map[string]struct{}) + dth.evictedBuffer = make(map[string]common.Trie) +} + +// IsInterfaceNil returns true if underlying object is nil +func (dth *dataTriesHolder) IsInterfaceNil() bool { + return dth == nil +} diff --git a/state/triesHolder/dataTriesHolder_test.go b/state/triesHolder/dataTriesHolder_test.go new file mode 100644 index 00000000000..20217f470fa --- /dev/null +++ b/state/triesHolder/dataTriesHolder_test.go @@ -0,0 +1,474 @@ +package triesHolder + +import ( + "errors" + "strconv" + "sync" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" + trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/stretchr/testify/assert" +) + +const ( + dthSize = 2 * 1024 * 1024 // 2MB + oneKB = 1 * 1024 +) + +type testTries struct { + key []byte + trie common.Trie +} + +func getTestTries(numTries int) []testTries { + tries := make([]testTries, 0) + for i := 0; i < numTries; i++ { + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return oneKB + }, + } + key := []byte("trie" + strconv.Itoa(i)) + tries = append(tries, testTries{key: key, trie: tr}) + } + return tries +} + +func TestNewDataTriesHolder(t *testing.T) { + t.Parallel() + + t.Run(" invalid max size", func(t *testing.T) { + t.Parallel() + + dth, err := NewDataTriesHolder(512 * 1024) // less than 1MB + assert.True(t, errors.Is(err, ErrInvalidMaxTrieSizeValue)) + assert.True(t, check.IfNil(dth)) + }) + + t.Run("should create new instance", func(t *testing.T) { + t.Parallel() + + dth, err := NewDataTriesHolder(dthSize) + assert.Nil(t, err) + assert.False(t, check.IfNil(dth)) + assert.NotNil(t, dth.evictedBuffer) + assert.NotNil(t, dth.dirtyTries) + assert.NotNil(t, dth.touchedTries) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, dth.cacher.Len()) + }) +} + +func TestDataTriesHolder_Put(t *testing.T) { + t.Parallel() + + t.Run("put invalid data", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + + dth.Put([]byte("key"), nil) + dth.Put(nil, &trieMock.TrieStub{}) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, dth.cacher.Len()) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, 0, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + }) + t.Run("put in empty tries holder", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + entry := getTestTries(1)[0] + + dth.Put(entry.key, entry.trie) + + assert.Equal(t, 1, dth.cacher.Len()) + assert.Equal(t, 1, len(dth.dirtyTries)) + retrievedEntry, ok := dth.cacher.Get(string(entry.key)) + assert.True(t, ok) + tr, ok := retrievedEntry.(common.Trie) + assert.True(t, ok) + assert.Equal(t, tr, entry.trie) + assert.Equal(t, uint64(oneKB), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 1, len(dth.dirtyTries)) + assert.Equal(t, 1, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + }) + t.Run("put in populated tries holder", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + cacherKeys := dth.cacher.Keys() + assert.Equal(t, numEntries, len(cacherKeys)) + for i := 0; i < numEntries; i++ { + retrievedEntry, ok := dth.cacher.Get(cacherKeys[i]) + assert.True(t, ok) + tr, ok := retrievedEntry.(common.Trie) + assert.True(t, ok) + assert.Equal(t, tr, entries[i].trie) + assert.Equal(t, cacherKeys[i], string(entries[i].key)) + } + }) + t.Run("put oldest used trie moves to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dth.Put(entries[0].key, entries[0].trie) + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[0].key), keys[numEntries-1]) + }) + t.Run("put existing trie moves to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + triePos := 2 + dth.Put(entries[triePos].key, entries[triePos].trie) + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[triePos].key), keys[numEntries-1]) + }) + t.Run("put with eviction - evicted dirty tries should be in eviction buffer", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return dthSize + }, + } + key := []byte("trieEvict") + + dth.Put(key, tr) + + assert.Equal(t, 1, dth.cacher.Len()) + assert.Equal(t, numEntries+1, len(dth.dirtyTries)) + assert.Equal(t, numEntries+1, len(dth.touchedTries)) + assert.Equal(t, numEntries, len(dth.evictedBuffer)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) + }) + t.Run("put with eviction - not dirty tries should evict", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries + + sizeToEvictTwoTries := dthSize - (3 * oneKB) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return sizeToEvictTwoTries + }, + } + key := []byte("trieEvict") + + dth.Put(key, tr) + numEvictedTries := 2 + + assert.Equal(t, numEntries-numEvictedTries+1, dth.cacher.Len()) + assert.Equal(t, 1, len(dth.dirtyTries)) + assert.Equal(t, numEntries+1, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) + }) +} + +func TestDataTriesHolder_Get(t *testing.T) { + t.Parallel() + + t.Run("get not existing trie should return nil", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + tr := dth.Get([]byte("notExistingKey")) + assert.Nil(t, tr) + assert.Equal(t, 0, dth.cacher.Len()) + }) + t.Run("get existing trie should move to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + tr := dth.Get(entries[1].key) + assert.Equal(t, entries[1].trie, tr) + assert.Equal(t, numEntries, dth.cacher.Len()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[1].key), keys[numEntries-1]) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + }) + t.Run("get from evicted buffer should put back in cache", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, 0, len(dth.evictedBuffer)) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return dthSize + }, + } + key := []byte("trieEvict") + dth.Put(key, tr) + assert.Equal(t, numEntries, len(dth.evictedBuffer)) + assert.Equal(t, 1, dth.cacher.Len()) + + numGetFromTrie := 3 + for i := 0; i < numGetFromTrie; i++ { + _ = dth.Get(entries[i].key) + } + + assert.Equal(t, 3, len(dth.evictedBuffer)) // 2 original tries + trieEvict + assert.Equal(t, numGetFromTrie, dth.cacher.Len()) + }) +} + +func TestDataTriesHolder_GetAll(t *testing.T) { + t.Parallel() + + t.Run("dirty trie not found in tries map does not panic", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.cacher.Remove(string(entries[0].key)) + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries-1, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + }) + t.Run("trie size is correctly computed after GetAll and eviction", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + + trieSize := dthSize - ((numEntries + 1) * oneKB) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return trieSize + }, + } + key := []byte("newTrie") + dth.Put(key, tr) + assert.Equal(t, numEntries+1, dth.cacher.Len()) + assert.Equal(t, uint64(numEntries*oneKB+trieSize), dth.cacher.SizeInBytesContained()) + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries + + // get a trie and "resolve" some nodes, thus increasing its size in memory + _ = dth.Get(key) + originalSize := trieSize + trieSize = trieSize + oneKB + assert.Equal(t, uint64(numEntries*oneKB+originalSize), dth.cacher.SizeInBytesContained()) // size is not updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + assert.Equal(t, 1, len(dth.touchedTries)) + + dirtyTries = dth.GetAll() + assert.Equal(t, 0, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) // size is updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + assert.Equal(t, 0, len(dth.touchedTries)) + + // put again the same trie, now with increased size triggering eviction + _ = dth.Get(key) + assert.Equal(t, 1, len(dth.touchedTries)) + trieSize = trieSize + oneKB/2 + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) // size is not updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + + dirtyTries = dth.GetAll() + assert.Equal(t, 0, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize-oneKB/2), dth.cacher.SizeInBytesContained()) // size is updated + assert.Equal(t, numEntries, dth.cacher.Len()) // eviction + assert.Equal(t, 0, len(dth.touchedTries)) + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + }) +} + +func TestDataTriesHolder_Reset(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dth.Reset() + assert.Equal(t, 0, dth.cacher.Len()) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, 0, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) +} + +func TestDataTriesHolder_Concurrency(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + + wg := sync.WaitGroup{} + wg.Add(numEntries) + + for i := 0; i < numEntries; i++ { + go func(key int) { + dth.Put(entries[key].key, entries[key].trie) + wg.Done() + }(i) + } + + wg.Wait() + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) +} + +func BenchmarkDataTriesHolder_PutWithEviction(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB / 10)) // set max size to 10% of total size to force evictions + entries := getTestTries(numEntries) + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Put(entry.key, entry.trie) + if i%1000 == 0 { + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries to allow evictions + } + } +} + +func BenchmarkDataTriesHolder_PutNoEviction(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) // set max size to 200% of total size to avoid evictions + entries := getTestTries(numEntries) + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Put(entry.key, entry.trie) + } +} + +func BenchmarkDataTriesHolder_Get(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Get(entry.key) + } +} + +func BenchmarkDataTriesHolder_GetAll(b *testing.B) { + numEntries := 10000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.dirtyTries = make(map[string]struct{}) + dth.touchedTries = make(map[string]struct{}) + numDirty := numEntries / 10 + for i := 0; i < numDirty; i++ { + dth.dirtyTries[string(entries[i].key)] = struct{}{} + dth.touchedTries[string(entries[i].key)] = struct{}{} + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = dth.GetAll() + } +} diff --git a/state/triesHolder/errors.go b/state/triesHolder/errors.go new file mode 100644 index 00000000000..539ef76c3f5 --- /dev/null +++ b/state/triesHolder/errors.go @@ -0,0 +1,6 @@ +package triesHolder + +import "errors" + +// ErrInvalidMaxTrieSizeValue signals that the provided max trie size value is invalid +var ErrInvalidMaxTrieSizeValue = errors.New("invalid max trie size value") diff --git a/state/dataTriesHolder.go b/state/triesHolder/triesHolder.go similarity index 51% rename from state/dataTriesHolder.go rename to state/triesHolder/triesHolder.go index 8333b875fce..75f1023a5f2 100644 --- a/state/dataTriesHolder.go +++ b/state/triesHolder/triesHolder.go @@ -1,4 +1,4 @@ -package state +package triesHolder import ( "sync" @@ -7,34 +7,29 @@ import ( logger "github.com/multiversx/mx-chain-logger-go" ) -type dataTriesHolder struct { +type triesHolder struct { tries map[string]common.Trie mutex sync.RWMutex } -// NewDataTriesHolder creates a new instance of dataTriesHolder -func NewDataTriesHolder() *dataTriesHolder { - return &dataTriesHolder{ +// NewTriesHolder creates a new instance of triesHolder +func NewTriesHolder() *triesHolder { + return &triesHolder{ tries: make(map[string]common.Trie), } } // Put adds a trie pointer to the tries map -func (dth *dataTriesHolder) Put(key []byte, tr common.Trie) { - log.Trace("put trie in data tries holder", "key", key) +func (dth *triesHolder) Put(key []byte, tr common.Trie) { + log.Trace("put trie in tries holder", "key", key) dth.mutex.Lock() dth.tries[string(key)] = tr dth.mutex.Unlock() } -// Replace changes a trie pointer to the tries map -func (dth *dataTriesHolder) Replace(key []byte, tr common.Trie) { - dth.Put(key, tr) -} - // Get returns the trie pointer that is stored in the map at the given key -func (dth *dataTriesHolder) Get(key []byte) common.Trie { +func (dth *triesHolder) Get(key []byte) common.Trie { dth.mutex.Lock() defer dth.mutex.Unlock() @@ -42,7 +37,7 @@ func (dth *dataTriesHolder) Get(key []byte) common.Trie { } // GetAll returns all trie pointers from the map -func (dth *dataTriesHolder) GetAll() []common.Trie { +func (dth *triesHolder) GetAll() []common.Trie { dth.mutex.Lock() defer dth.mutex.Unlock() @@ -54,21 +49,8 @@ func (dth *dataTriesHolder) GetAll() []common.Trie { return tries } -// GetAllTries returns the tries with key value map -func (dth *dataTriesHolder) GetAllTries() map[string]common.Trie { - dth.mutex.Lock() - defer dth.mutex.Unlock() - - copyTries := make(map[string]common.Trie, len(dth.tries)) - for key, trie := range dth.tries { - copyTries[key] = trie - } - - return copyTries -} - // Reset clears the tries map -func (dth *dataTriesHolder) Reset() { +func (dth *triesHolder) Reset() { dth.mutex.Lock() if log.GetLevel() == logger.LogTrace { @@ -82,6 +64,6 @@ func (dth *dataTriesHolder) Reset() { } // IsInterfaceNil returns true if underlying object is nil -func (dth *dataTriesHolder) IsInterfaceNil() bool { +func (dth *triesHolder) IsInterfaceNil() bool { return dth == nil } diff --git a/state/triesHolder/triesHolder_test.go b/state/triesHolder/triesHolder_test.go new file mode 100644 index 00000000000..f34f0ef6dfe --- /dev/null +++ b/state/triesHolder/triesHolder_test.go @@ -0,0 +1,88 @@ +package triesHolder + +import ( + "strconv" + "sync" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/stretchr/testify/assert" +) + +func TestNewTriesHolder(t *testing.T) { + t.Parallel() + + dth := NewTriesHolder() + assert.False(t, check.IfNil(dth)) +} + +func TestTriesHolder_PutAndGet(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + tr := dth.Get([]byte("trie1")) + + assert.True(t, tr == tr1) +} + +func TestTriesHolder_GetAll(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + tr2 := &trieMock.TrieStub{} + tr3 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + dth.Put([]byte("trie2"), tr2) + dth.Put([]byte("trie3"), tr3) + tries := dth.GetAll() + + assert.Equal(t, 3, len(tries)) +} + +func TestTriesHolder_Reset(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + dth.Reset() + + tr := dth.Get([]byte("trie1")) + assert.Nil(t, tr) +} + +func TestTriesHolder_Concurrency(t *testing.T) { + t.Parallel() + + dth := NewTriesHolder() + numCalls := 5000 + + wg := sync.WaitGroup{} + wg.Add(numCalls) + + for i := 0; i < numCalls; i++ { + go func(key int) { + defer wg.Done() + + switch key % 4 { + case 0: + dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) + case 1: + dth.Get([]byte(strconv.Itoa(key))) + case 2: + dth.GetAll() + case 3: + dth.Reset() + } + }(i) + } + + wg.Wait() +} diff --git a/testscommon/storage/storageManagerArgs.go b/testscommon/common/storageManagerArgs.go similarity index 98% rename from testscommon/storage/storageManagerArgs.go rename to testscommon/common/storageManagerArgs.go index 1f32e18f0d0..6e244bfb162 100644 --- a/testscommon/storage/storageManagerArgs.go +++ b/testscommon/common/storageManagerArgs.go @@ -1,4 +1,4 @@ -package storage +package common import ( "github.com/multiversx/mx-chain-go/common/statistics/disabled" diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 8ef763a7cc7..e12700267db 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-go/state/triesHolder" logger "github.com/multiversx/mx-chain-logger-go" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" "github.com/stretchr/testify/require" @@ -37,13 +38,11 @@ import ( p2pConfig "github.com/multiversx/mx-chain-go/p2p/config" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/dblookupext" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -364,20 +363,21 @@ func GetNetworkFactoryArgs() networkComp.NetworkComponentsFactoryArgs { // GetStateFactoryArgs - func GetStateFactoryArgs(coreComponents factory.CoreComponentsHolder, statusCoreComp factory.StatusCoreComponentsHolder) stateComp.StateComponentsFactoryArgs { - tsm, _ := trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) + tsm, _ := trie.NewTrieStorageManager(commonMocks.GetStorageManagerArgs()) storageManagerUser, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) - tsm, _ = trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) + tsm, _ = trie.NewTrieStorageManager(commonMocks.GetStorageManagerArgs()) storageManagerPeer, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer - triesHolder := state.NewDataTriesHolder() - trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler()) - triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler()) - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) + triesContainer := triesHolder.NewTriesHolder() + tenMBSize := uint64(10485760) + trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), tenMBSize) + triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), tenMBSize) + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) stateComponentsFactoryArgs := stateComp.StateComponentsFactoryArgs{ Config: GetGeneralConfig(), diff --git a/testscommon/components/configs.go b/testscommon/components/configs.go index ef63c53dc4f..c5d08a3ba3b 100644 --- a/testscommon/components/configs.go +++ b/testscommon/components/configs.go @@ -5,6 +5,8 @@ import ( "github.com/multiversx/mx-chain-go/testscommon" ) +const tenMBSize = uint64(10485760) + // GetGeneralConfig - func GetGeneralConfig() config.Config { return config.Config{ @@ -22,6 +24,9 @@ func GetGeneralConfig() config.Config { StateTriesConfig: config.StateTriesConfig{ AccountsStatePruningEnabled: true, PeerStatePruningEnabled: true, + MaxUserTrieSizeInMemory: tenMBSize, + MaxPeerTrieSizeInMemory: tenMBSize, + DataTriesSizeInMemory: tenMBSize, }, EvictionWaitingList: config.EvictionWaitingListConfig{ HashesSize: 100, diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index a5ab9c93a24..7e7b5f1e3b5 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -5,6 +5,8 @@ import ( "github.com/multiversx/mx-chain-go/storage/storageunit" ) +const tenMbSize = uint64(10485760) + // GetGeneralConfig returns the common configuration used for testing func GetGeneralConfig() config.Config { return config.Config{ @@ -152,6 +154,9 @@ func GetGeneralConfig() config.Config { SnapshotsEnabled: true, AccountsStatePruningEnabled: false, PeerStatePruningEnabled: false, + MaxUserTrieSizeInMemory: tenMbSize, + MaxPeerTrieSizeInMemory: tenMbSize, + DataTriesSizeInMemory: tenMbSize, }, TrieStorageManagerConfig: config.TrieStorageManagerConfig{ PruningBufferLen: 1000, diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index 827552011c6..aca3bdfdd34 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -18,9 +18,9 @@ import ( "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" testStorage "github.com/multiversx/mx-chain-go/testscommon/state" - testcommonStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -91,14 +91,15 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testcommonStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = db args.Marshalizer = TestMarshalizer args.Hasher = TestHasher trieStorage, _ := trie.NewTrieStorageManager(args) + tenMBSize := uint64(10485760) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, enableEpochs) + tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, enableEpochs, tenMBSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := accountFactory.ArgsAccountCreator{ @@ -122,14 +123,15 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: TestHasher, - Marshaller: TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: TestHasher, + Marshaller: TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/testscommon/state/testTrie.go b/testscommon/state/testTrie.go index 12c5f58b3f0..81e4cc77117 100644 --- a/testscommon/state/testTrie.go +++ b/testscommon/state/testTrie.go @@ -40,7 +40,8 @@ func GetDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, has // GetNewTrie - func GetNewTrie() common.Trie { tsm, marshaller, hasher := GetDefaultTrieParameters() - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr } diff --git a/testscommon/trie/snapshotPruningStorerStub.go b/testscommon/storage/snapshotPruningStorerStub.go similarity index 99% rename from testscommon/trie/snapshotPruningStorerStub.go rename to testscommon/storage/snapshotPruningStorerStub.go index 8de709ab2cd..f6186eb49dc 100644 --- a/testscommon/trie/snapshotPruningStorerStub.go +++ b/testscommon/storage/snapshotPruningStorerStub.go @@ -1,4 +1,4 @@ -package trie +package storage import ( "github.com/multiversx/mx-chain-core-go/core" diff --git a/testscommon/trie/triesHolderStub.go b/testscommon/trie/triesHolderStub.go index 42eab41d7f5..4d5a57e5738 100644 --- a/testscommon/trie/triesHolderStub.go +++ b/testscommon/trie/triesHolderStub.go @@ -20,13 +20,6 @@ func (ths *TriesHolderStub) Put(key []byte, trie common.Trie) { } } -// Replace - -func (ths *TriesHolderStub) Replace(key []byte, trie common.Trie) { - if ths.RemoveCalled != nil { - ths.RemoveCalled(key, trie) - } -} - // Get - func (ths *TriesHolderStub) Get(key []byte) common.Trie { if ths.GetCalled != nil { diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 3bee219adb0..0fd347ca79e 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" ) +const tenMBSize = uint64(10485760) + var dtmc = trieMetricsCollector.NewDisabledTrieMetricsCollector() func getTestMarshalizerAndHasher() (marshal.Marshalizer, hashing.Hasher) { @@ -95,6 +97,7 @@ func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) { oldRoot: make([]byte, 0), chanClose: make(chan struct{}), enableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + maxSizeInMem: tenMBSize, } return tr, trieStorage @@ -209,8 +212,8 @@ func TestBranchNode_setRootHash(t *testing.T) { trieStorage1, _ := NewTrieStorageManager(GetDefaultTrieStorageManagerParameters()) trieStorage2, _ := NewTrieStorageManager(GetDefaultTrieStorageManagerParameters()) - tr1, _ := NewTrie(trieStorage1, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) - tr2, _ := NewTrie(trieStorage2, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr1, _ := NewTrie(trieStorage1, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) + tr2, _ := NewTrie(trieStorage2, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) maxIterations := 10000 for i := 0; i < maxIterations; i++ { diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 99eeedd29b0..41ebc3cdd77 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -37,7 +37,7 @@ func createTrieStorageManager(store storage.Storer) (common.StorageManager, stor func createInMemoryTrie() (common.Trie, storage.Storer) { memUnit := testscommon.CreateMemUnit() tsm, _ := createTrieStorageManager(memUnit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr, memUnit } @@ -50,7 +50,7 @@ func createInMemoryTrieFromDB(db storage.Persister) (common.Trie, storage.Storer unit, _ := storageunit.NewStorageUnit(cache, db) tsm, _ := createTrieStorageManager(unit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr, unit } diff --git a/trie/errors.go b/trie/errors.go index a879fd6c94c..54ec2ec1d3e 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -129,3 +129,6 @@ var ErrEmptyInitialIteratorState = errors.New("empty initial iterator state") // ErrInvalidIteratorState signals that an invalid iterator state was provided var ErrInvalidIteratorState = errors.New("invalid iterator state") + +// ErrInvalidMaxSizeInMemory signals that the provided max trie size value is invalid +var ErrInvalidMaxSizeInMemory = errors.New("invalid max size in memory") diff --git a/trie/factory/trieCreator.go b/trie/factory/trieCreator.go index 677f7fb2193..621fbfb0e8b 100644 --- a/trie/factory/trieCreator.go +++ b/trie/factory/trieCreator.go @@ -7,7 +7,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -21,6 +21,7 @@ type TrieCreateArgs struct { Identifier string EnableEpochsHandler common.EnableEpochsHandler StatsCollector common.StateStatisticsHandler + MaxSizeInMemory uint64 } type trieCreator struct { @@ -77,7 +78,7 @@ func (tc *trieCreator) Create(args TrieCreateArgs) (common.StorageManager, commo return nil, nil, err } - newTrie, err := trie.NewTrie(trieStorage, tc.marshalizer, tc.hasher, args.EnableEpochsHandler) + newTrie, err := trie.NewTrie(trieStorage, tc.marshalizer, tc.hasher, args.EnableEpochsHandler, args.MaxSizeInMemory) if err != nil { return nil, nil, err } @@ -121,13 +122,14 @@ func CreateTriesComponentsForShardId( Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + MaxSizeInMemory: generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory, } userStorageManager, userAccountTrie, err := trFactory.Create(args) if err != nil { return nil, nil, err } - trieContainer := state.NewDataTriesHolder() + trieContainer := triesHolder.NewTriesHolder() trieStorageManagers := make(map[string]common.StorageManager) trieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), userAccountTrie) @@ -146,6 +148,7 @@ func CreateTriesComponentsForShardId( Identifier: dataRetriever.PeerAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + MaxSizeInMemory: generalConfig.StateTriesConfig.MaxPeerTrieSizeInMemory, } peerStorageManager, peerAccountsTrie, err := trFactory.Create(args) if err != nil { diff --git a/trie/factory/trieCreator_test.go b/trie/factory/trieCreator_test.go index 83e2a8f9500..e67b715be02 100644 --- a/trie/factory/trieCreator_test.go +++ b/trie/factory/trieCreator_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -40,6 +41,7 @@ func getCreateArgs() factory.TrieCreateArgs { Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, StatsCollector: disabled.NewStateStatistics(), + MaxSizeInMemory: common.TenMbSize, } } diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 11020227492..8b744ee4dd0 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -30,6 +30,7 @@ const ( extension = iota leaf branch + minSizeInMemory = 1048576 // 1 MB ) type patriciaMerkleTrie struct { @@ -46,6 +47,7 @@ type patriciaMerkleTrie struct { oldRoot []byte chanClose chan struct{} sizeInMemory int + maxSizeInMem uint64 } // NewTrie creates a new Patricia Merkle Trie @@ -54,6 +56,7 @@ func NewTrie( msh marshal.Marshalizer, hsh hashing.Hasher, enableEpochsHandler common.EnableEpochsHandler, + maxSizeInMemory uint64, ) (*patriciaMerkleTrie, error) { if check.IfNil(trieStorage) { return nil, ErrNilTrieStorage @@ -67,6 +70,9 @@ func NewTrie( if check.IfNil(enableEpochsHandler) { return nil, errors.ErrNilEnableEpochsHandler } + if maxSizeInMemory < minSizeInMemory { + return nil, fmt.Errorf("%w: provided %d, minimum %d", ErrInvalidMaxSizeInMemory, maxSizeInMemory, minSizeInMemory) + } tnvv, err := core.NewTrieNodeVersionVerifier(enableEpochsHandler) if err != nil { @@ -82,6 +88,8 @@ func NewTrie( chanClose: make(chan struct{}), enableEpochsHandler: enableEpochsHandler, trieNodeVersionVerifier: tnvv, + // TODO collapse trie leaves if maxSizeInMemory is reached + maxSizeInMem: maxSizeInMemory, }, nil } @@ -295,6 +303,7 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( tr.marshalizer, tr.hasher, tr.enableEpochsHandler, + tr.maxSizeInMem, ) } @@ -375,6 +384,7 @@ func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.Storage tr.marshalizer, tr.hasher, tr.enableEpochsHandler, + tr.maxSizeInMem, ) if err != nil { return nil, nil, err diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 4455d215f82..7f9361638ae 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -37,6 +37,8 @@ import ( var emptyTrieHash = make([]byte, 32) +const tenMBSize = uint64(10485760) + func emptyTrie() common.Trie { tr, _ := trie.NewTrie(getDefaultTrieParameters()) @@ -44,17 +46,17 @@ func emptyTrie() common.Trie { } func emptyTrieWithCustomEnableEpochsHandler(handler common.EnableEpochsHandler) common.Trie { - storage, marshaller, hasher, _ := getDefaultTrieParameters() + storage, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() - tr, _ := trie.NewTrie(storage, marshaller, hasher, handler) + tr, _ := trie.NewTrie(storage, marshaller, hasher, handler, maxSizeInMem) return tr } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler) { +func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint64) { args := trie.GetDefaultTrieStorageManagerParameters() trieStorageManager, _ := trie.NewTrieStorageManager(args) - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{} + return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize } func initTrieMultipleValues(nr int) (common.Trie, [][]byte) { @@ -87,8 +89,8 @@ func addDefaultDataToTrie(tr common.Trie) { func TestNewTrieWithNilTrieStorage(t *testing.T) { t.Parallel() - _, marshalizer, hasher, enableEpochsHandler := getDefaultTrieParameters() - tr, err := trie.NewTrie(nil, marshalizer, hasher, enableEpochsHandler) + _, marshalizer, hasher, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(nil, marshalizer, hasher, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilTrieStorage, err) @@ -97,8 +99,8 @@ func TestNewTrieWithNilTrieStorage(t *testing.T) { func TestNewTrieWithNilMarshalizer(t *testing.T) { t.Parallel() - trieStorage, _, hasher, enableEpochsHandler := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, nil, hasher, enableEpochsHandler) + trieStorage, _, hasher, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, nil, hasher, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilMarshalizer, err) @@ -107,8 +109,8 @@ func TestNewTrieWithNilMarshalizer(t *testing.T) { func TestNewTrieWithNilHasher(t *testing.T) { t.Parallel() - trieStorage, marshalizer, _, enableEpochsHandler := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, nil, enableEpochsHandler) + trieStorage, marshalizer, _, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, marshalizer, nil, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilHasher, err) @@ -117,13 +119,23 @@ func TestNewTrieWithNilHasher(t *testing.T) { func TestNewTrieWithNilEnableEpochsHandler(t *testing.T) { t.Parallel() - trieStorage, marshalizer, hasher, _ := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, nil) + trieStorage, marshalizer, hasher, _, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, nil, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, errorsCommon.ErrNilEnableEpochsHandler, err) } +func TestNewTrieWithInvalidMaxSizeInMemory(t *testing.T) { + t.Parallel() + + trieStorage, marshalizer, hasher, enableEpochsHandler, _ := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, enableEpochsHandler, 0) + + assert.Nil(t, tr) + assert.True(t, errors.Is(err, trie.ErrInvalidMaxSizeInMemory)) +} + func TestPatriciaMerkleTree_Get(t *testing.T) { t.Parallel() @@ -1052,7 +1064,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodesShouldSerializeTheCalls(t *testing }, } - tr, _ := trie.NewTrie(testTrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}) + tr, _ := trie.NewTrie(testTrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) numGoRoutines := 100 wg := sync.WaitGroup{} wg.Add(numGoRoutines) @@ -1474,13 +1486,13 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("not migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _ := getDefaultTrieParameters() + tsm, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxSizeInMem) _ = tr.Update([]byte("dog"), []byte("reindeer")) isMigrated, err := tr.IsMigratedToLatestVersion() @@ -1491,13 +1503,13 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _ := getDefaultTrieParameters() + tsm, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxSizeInMem) _ = tr.UpdateWithVersion([]byte("dog"), []byte("reindeer"), core.AutoBalanceEnabled) isMigrated, err := tr.IsMigratedToLatestVersion() diff --git a/trie/snapshotTrieStorageManager_test.go b/trie/snapshotTrieStorageManager_test.go index c8cc2df3ce2..9415edd6566 100644 --- a/trie/snapshotTrieStorageManager_test.go +++ b/trie/snapshotTrieStorageManager_test.go @@ -10,7 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/trie" + storageMock "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,7 @@ func TestNewSnapshotTrieStorageManager(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{} + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{} stsm, err := newSnapshotTrieStorageManager(trieStorage, 0) assert.Nil(t, err) assert.False(t, check.IfNil(stsm)) @@ -43,7 +43,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(key []byte, maxEpochToSearchFrom uint32) ([]byte, core.OptionalUint32, error) { return nil, core.OptionalUint32{}, core.ErrContextClosing }, @@ -59,7 +59,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { return nil, core.OptionalUint32{}, storage.ErrDBIsClosed }, @@ -75,7 +75,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { _, trieStorage := newEmptyTrie() getFromOldEpochsWithoutCacheCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { getFromOldEpochsWithoutCacheCalled = true return nil, core.OptionalUint32{}, nil @@ -95,7 +95,7 @@ func TestSnapshotTrieStorageManager_PutInEpochWithoutCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ PutInEpochWithoutCacheCalled: func(_ []byte, _ []byte, _ uint32) error { return core.ErrContextClosing }, @@ -111,7 +111,7 @@ func TestSnapshotTrieStorageManager_PutInEpochWithoutCache(t *testing.T) { _, trieStorage := newEmptyTrie() putWithoutCacheCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ PutInEpochWithoutCacheCalled: func(_ []byte, _ []byte, _ uint32) error { putWithoutCacheCalled = true return nil @@ -131,7 +131,7 @@ func TestSnapshotTrieStorageManager_GetFromLastEpoch(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetFromLastEpochCalled: func(_ []byte) ([]byte, error) { return nil, core.ErrContextClosing }, @@ -148,7 +148,7 @@ func TestSnapshotTrieStorageManager_GetFromLastEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromLastEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetFromLastEpochCalled: func(_ []byte) ([]byte, error) { getFromLastEpochCalled = true return nil, nil @@ -167,7 +167,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("HasValue is false", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { return val, core.OptionalUint32{}, nil }, @@ -184,7 +184,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("epoch is previous epoch", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 4, @@ -205,7 +205,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("epoch is 0", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 4, @@ -226,7 +226,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("key is ActiveDBKey", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, @@ -247,7 +247,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("key is TrieSyncedKey", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, @@ -269,7 +269,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { val := []byte("val") putInEpochCalled := false _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, diff --git a/trie/syncTrieStorageManager_test.go b/trie/syncTrieStorageManager_test.go index 0e7c7532433..5e7fe9ce3e5 100644 --- a/trie/syncTrieStorageManager_test.go +++ b/trie/syncTrieStorageManager_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" ) @@ -34,7 +34,7 @@ func TestNewSyncTrieStorageManager(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{} + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{} stsm, err := NewSyncTrieStorageManager(trieStorage) assert.Nil(t, err) assert.NotNil(t, stsm) @@ -45,7 +45,7 @@ func TestNewSyncTrieStorageManager_PutInFirstEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() putInEpochCalled := 0 - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { putInEpochCalled++ return nil @@ -66,7 +66,7 @@ func TestNewSyncTrieStorageManager_PutInEpochError(t *testing.T) { expectedErr := errors.New("expected error") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { return expectedErr }, @@ -82,7 +82,7 @@ func TestNewSyncTrieStorageManager_PutInEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() putInEpochCalled := 0 - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { putInEpochCalled++ return nil diff --git a/trie/sync_test.go b/trie/sync_test.go index 06a23fee091..28b957d2bb8 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" + trieMock "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,7 +20,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" - trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie/statistics" "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) diff --git a/trie/trieStorageManagerInEpoch_test.go b/trie/trieStorageManagerInEpoch_test.go index bedf3734529..ca0717242d2 100644 --- a/trie/trieStorageManagerInEpoch_test.go +++ b/trie/trieStorageManagerInEpoch_test.go @@ -7,8 +7,8 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/storageManager" - "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/stretchr/testify/assert" ) @@ -72,7 +72,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, nil @@ -89,7 +89,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, storage.ErrDBIsClosed @@ -107,7 +107,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, errors.New("not closing error") @@ -128,7 +128,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { getFromPreviousEpochCalled := false currentEpoch := uint32(5) expectedKey := []byte("key") - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(key []byte, epoch uint32) ([]byte, error) { assert.Equal(t, expectedKey, key) if epoch == currentEpoch { diff --git a/trie/trieStorageManager_test.go b/trie/trieStorageManager_test.go index 104c6e2578c..88c9bb7888a 100644 --- a/trie/trieStorageManager_test.go +++ b/trie/trieStorageManager_test.go @@ -207,7 +207,7 @@ func TestTrieStorageManager_RemoveFromAllActiveEpochs(t *testing.T) { RemoveFromAllActiveEpochsCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), RemoveFromAllActiveEpochsCalled: func(key []byte) error { RemoveFromAllActiveEpochsCalled = true @@ -237,7 +237,7 @@ func TestTrieStorageManager_PutInEpoch(t *testing.T) { putInEpochCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), PutInEpochCalled: func(key []byte, data []byte, epoch uint32) error { putInEpochCalled = true @@ -268,7 +268,7 @@ func TestTrieStorageManager_GetLatestStorageEpoch(t *testing.T) { getLatestSorageCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), GetLatestStorageEpochCalled: func() (uint32, error) { getLatestSorageCalled = true @@ -384,7 +384,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return []byte(common.TrieSyncedVal), nil }, @@ -398,7 +398,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return []byte("invalid marker"), nil }, @@ -412,7 +412,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return nil, expectedErr // isTrieSynced returns false }, @@ -477,7 +477,7 @@ func TestNewSnapshotTrieStorageManager_GetFromCurrentEpoch(t *testing.T) { getFromCurrentEpochCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), GetFromCurrentEpochCalled: func(_ []byte) ([]byte, error) { getFromCurrentEpochCalled = true diff --git a/update/factory/dataTrieFactory.go b/update/factory/dataTrieFactory.go index 324fc4d0f38..78e66f5a651 100644 --- a/update/factory/dataTrieFactory.go +++ b/update/factory/dataTrieFactory.go @@ -13,7 +13,7 @@ import ( "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/trie" @@ -21,6 +21,8 @@ import ( "github.com/multiversx/mx-chain-go/update/genesis" ) +const tenMBSize = uint64(10485760) + // ArgsNewDataTrieFactory is the argument structure for the new data trie factory type ArgsNewDataTrieFactory struct { StorageConfig config.StorageConfig @@ -115,7 +117,7 @@ func (d *dataTrieFactory) TrieStorageManager() common.StorageManager { // Create creates a TriesHolder container to hold all the states func (d *dataTrieFactory) Create() (common.TriesHolder, error) { - container := state.NewDataTriesHolder() + container := triesHolder.NewTriesHolder() for i := uint32(0); i < d.shardCoordinator.NumberOfShards(); i++ { err := d.createAndAddOneTrie(i, genesis.UserAccount, container) @@ -138,7 +140,7 @@ func (d *dataTrieFactory) Create() (common.TriesHolder, error) { } func (d *dataTrieFactory) createAndAddOneTrie(shId uint32, accType genesis.Type, container common.TriesHolder) error { - dataTrie, err := trie.NewTrie(d.trieStorage, d.marshalizer, d.hasher, d.enableEpochsHandler) + dataTrie, err := trie.NewTrie(d.trieStorage, d.marshalizer, d.hasher, d.enableEpochsHandler, tenMBSize) if err != nil { return err } diff --git a/update/genesis/import.go b/update/genesis/import.go index 3455a60d4e6..dd1984081af 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -25,6 +25,8 @@ import ( "github.com/multiversx/mx-chain-go/update" ) +const tenMBSize = uint64(10485760) + var _ update.ImportHandler = (*stateImport)(nil) // ArgsNewStateImport is the arguments structure to create a new state importer @@ -313,7 +315,7 @@ func (si *stateImport) getTrie(shardID uint32, accType Type) (common.Trie, error trieStorageManager = si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] } - trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, si.enableEpochsHandler) + trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, si.enableEpochsHandler, tenMBSize) if err != nil { return nil, err } @@ -345,7 +347,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w wanted a roothash", update.ErrWrongTypeAssertion) } - dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, si.enableEpochsHandler) + dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, si.enableEpochsHandler, tenMBSize) if err != nil { return err } @@ -412,14 +414,15 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32, accountFactor if accType == ValidatorAccount { if check.IfNil(si.validatorDB) { argsAccountDB := state.ArgsAccountsDB{ - Trie: currentTrie, - Hasher: si.hasher, - Marshaller: si.marshalizer, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: si.addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: si.addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } accountsDB, errCreate := state.NewAccountsDB(argsAccountDB) if errCreate != nil { @@ -436,14 +439,15 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32, accountFactor } argsAccountDB := state.ArgsAccountsDB{ - Trie: currentTrie, - Hasher: si.hasher, - Marshaller: si.marshalizer, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: si.addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: si.addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } accountsDB, err = state.NewAccountsDB(argsAccountDB) si.accountDBsMap[shardID] = accountsDB