Skip to content

dataTriesHolder with eviction cache #7191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BeniaminDrasovean merged 13 commits into from
Mar 16, 2026
Merged

dataTriesHolder with eviction cache #7191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
65c8aa5
dataTriesHolder as linked list
BeniaminDrasovean Aug 14, 2025
b78c176
add eviction and unit tests
BeniaminDrasovean Sep 3, 2025
621f6f9
keep totalSizeInMemory updated between commits
BeniaminDrasovean Sep 4, 2025
c55760b
add size limits to config
BeniaminDrasovean Oct 2, 2025
d4eb6a2
create dataTriesHolder with size read from config
BeniaminDrasovean Oct 8, 2025
a1a9635
use existing cacher instead of linked list
BeniaminDrasovean Oct 14, 2025
5bfd75f
fix cyclic imports
BeniaminDrasovean Oct 14, 2025
dc8dc6e
add nil check for inserted data
BeniaminDrasovean Oct 14, 2025
beebe58
Merge branch 'feat/collapse-trie-based-on-size' into collapse-trie-ba…
BeniaminDrasovean Mar 3, 2026
19f9527
fix after merge
BeniaminDrasovean Mar 3, 2026
ad74409
fix linter issues
BeniaminDrasovean Mar 3, 2026
17870e2
fix typo in test
BeniaminDrasovean Mar 3, 2026
96ad288
fix after review
BeniaminDrasovean Mar 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmd/node/config/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,9 @@
AccountsStatePruningEnabled = false
PeerStatePruningEnabled = true
StateStatisticsEnabled = false
MaxUserTrieSizeInMemory = 524288000 #500MB
MaxPeerTrieSizeInMemory = 104857600 #100MB
DataTriesSizeInMemory = 524288000 #500MB

[TrieLeavesRetrieverConfig]
Enabled = false
Expand Down
3 changes: 3 additions & 0 deletions common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ const (
nonceIndex = 0
)

// TenMbSize defines the size of 10 megabytes in bytes, used as a constant for memory limits or buffer sizes
const TenMbSize = uint64(10485760)

type executionResultHandler interface {
GetMiniBlockHeadersHandlers() []data.MiniBlockHeaderHandler
}
Expand Down
1 change: 0 additions & 1 deletion common/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,6 @@ type SnapshotDbHandler interface {
// TriesHolder is used to store multiple tries
type TriesHolder interface {
Put([]byte, Trie)
Replace(key []byte, tr Trie)
Get([]byte) Trie
GetAll() []Trie
Reset()
Expand Down
3 changes: 3 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,9 @@ type StateTriesConfig struct {
AccountsStatePruningEnabled bool
PeerStatePruningEnabled bool
StateStatisticsEnabled bool
MaxUserTrieSizeInMemory uint64
MaxPeerTrieSizeInMemory uint64
DataTriesSizeInMemory uint64
}

// StateAccessesCollectorConfig will hold information about state accesses collector
Expand Down
10 changes: 5 additions & 5 deletions dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/multiversx/mx-chain-go/dataRetriever/mock"
"github.com/multiversx/mx-chain-go/p2p"
"github.com/multiversx/mx-chain-go/process/factory"
"github.com/multiversx/mx-chain-go/state"
"github.com/multiversx/mx-chain-go/state/triesHolder"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/testscommon"
"github.com/multiversx/mx-chain-go/testscommon/cache"
Expand Down Expand Up @@ -86,10 +86,10 @@ func createStoreForMeta() dataRetriever.StorageService {
}

func createTriesHolderForMeta() common.TriesHolder {
triesHolder := state.NewDataTriesHolder()
triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
return triesHolder
triesContainer := triesHolder.NewTriesHolder()
triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
return triesContainer
}

// ------- NewResolversContainerFactory
Expand Down
10 changes: 5 additions & 5 deletions dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/multiversx/mx-chain-go/dataRetriever/mock"
"github.com/multiversx/mx-chain-go/p2p"
"github.com/multiversx/mx-chain-go/process/factory"
"github.com/multiversx/mx-chain-go/state"
"github.com/multiversx/mx-chain-go/state/triesHolder"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/testscommon"
"github.com/multiversx/mx-chain-go/testscommon/cache"
Expand Down Expand Up @@ -92,10 +92,10 @@ func createStoreForShard() dataRetriever.StorageService {
}

func createTriesHolderForShard() common.TriesHolder {
triesHolder := state.NewDataTriesHolder()
triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
return triesHolder
triesContainer := triesHolder.NewTriesHolder()
triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
return triesContainer
}

// ------- NewResolversContainerFactory
Expand Down
7 changes: 3 additions & 4 deletions epochStart/bootstrap/process.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@ import (
"github.com/multiversx/mx-chain-core-go/data"
"github.com/multiversx/mx-chain-core-go/data/block"
"github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice"
"github.com/multiversx/mx-chain-go/state/triesHolder"
logger "github.com/multiversx/mx-chain-logger-go"

"github.com/multiversx/mx-chain-go/process/interceptors/processor"

"github.com/multiversx/mx-chain-go/common"
disabledCommon "github.com/multiversx/mx-chain-go/common/disabled"
"github.com/multiversx/mx-chain-go/common/ordering"
Expand All @@ -42,11 +41,11 @@ import (
"github.com/multiversx/mx-chain-go/process/heartbeat/validator"
"github.com/multiversx/mx-chain-go/process/interceptors"
disabledInterceptors "github.com/multiversx/mx-chain-go/process/interceptors/disabled"
"github.com/multiversx/mx-chain-go/process/interceptors/processor"
"github.com/multiversx/mx-chain-go/process/peer"
"github.com/multiversx/mx-chain-go/redundancy"
"github.com/multiversx/mx-chain-go/sharding"
"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
"github.com/multiversx/mx-chain-go/state"
"github.com/multiversx/mx-chain-go/state/syncer"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/storage/cache"
Expand Down Expand Up @@ -283,7 +282,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap,
return nil, err
}

epochStartProvider.trieContainer = state.NewDataTriesHolder()
epochStartProvider.trieContainer = triesHolder.NewTriesHolder()
epochStartProvider.trieStorageManagers = make(map[string]common.StorageManager)

if epochStartProvider.generalConfig.Hardfork.AfterHardFork {
Expand Down
3 changes: 3 additions & 0 deletions epochStart/bootstrap/process_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,9 @@ func createMockEpochStartBootstrapArgs(
AccountsStatePruningEnabled: true,
SnapshotsEnabled: true,
PeerStatePruningEnabled: true,
MaxUserTrieSizeInMemory: generalCfg.StateTriesConfig.MaxUserTrieSizeInMemory,
MaxPeerTrieSizeInMemory: generalCfg.StateTriesConfig.MaxPeerTrieSizeInMemory,
DataTriesSizeInMemory: generalCfg.StateTriesConfig.DataTriesSizeInMemory,
},
TrieStorageManagerConfig: config.TrieStorageManagerConfig{
PruningBufferLen: 1000,
Expand Down
5 changes: 2 additions & 3 deletions epochStart/metachain/baseRewards_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
"github.com/multiversx/mx-chain-go/testscommon/shardingMocks"
stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
"github.com/multiversx/mx-chain-go/testscommon/storage"
"github.com/multiversx/mx-chain-go/trie"
)

Expand Down Expand Up @@ -1140,11 +1139,11 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs {
hasher := sha256.NewSha256()
marshalizer := &marshal.GogoProtoMarshalizer{}

storageManagerArgs := storage.GetStorageManagerArgs()
storageManagerArgs := txExecOrderStub.GetStorageManagerArgs()
storageManagerArgs.Marshalizer = marshalizer
storageManagerArgs.Hasher = hasher

trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions())
trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, txExecOrderStub.GetStorageManagerOptions())
argsAccCreator := factory.ArgsAccountCreator{
Hasher: hasher,
Marshaller: marshalizer,
Expand Down
25 changes: 14 additions & 11 deletions epochStart/metachain/systemSCs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
storageFactory "github.com/multiversx/mx-chain-go/storage/factory"
"github.com/multiversx/mx-chain-go/storage/storageunit"
"github.com/multiversx/mx-chain-go/testscommon"
testCommon "github.com/multiversx/mx-chain-go/testscommon/common"
"github.com/multiversx/mx-chain-go/testscommon/cryptoMocks"
dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
"github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock"
Expand Down Expand Up @@ -748,7 +749,8 @@ func createAccountsDB(
trieStorageManager common.StorageManager,
enableEpochsHandler common.EnableEpochsHandler,
) *state.AccountsDB {
tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler)
tenMbSize := uint64(10485760)
tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, tenMbSize)
ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{
RootHashesSize: 100,
HashesSize: 10000,
Expand All @@ -757,14 +759,15 @@ func createAccountsDB(
spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10)

args := state.ArgsAccountsDB{
Trie: tr,
Hasher: hasher,
Marshaller: marshaller,
AccountFactory: accountFactory,
StoragePruningManager: spm,
AddressConverter: &testscommon.PubkeyConverterMock{},
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
Trie: tr,
Hasher: hasher,
Marshaller: marshaller,
AccountFactory: accountFactory,
StoragePruningManager: spm,
AddressConverter: &testscommon.PubkeyConverterMock{},
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
MaxDataTriesSizeInMemory: tenMbSize,
}
adb, _ := state.NewAccountsDB(args)
return adb
Expand All @@ -773,12 +776,12 @@ func createAccountsDB(
func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEpochs, trieStorer storage.Storer) (ArgsNewEpochStartSystemSCProcessing, vm.SystemSCContainer) {
hasher := sha256.NewSha256()
marshalizer := &marshal.GogoProtoMarshalizer{}
storageManagerArgs := storageMock.GetStorageManagerArgs()
storageManagerArgs := testCommon.GetStorageManagerArgs()
storageManagerArgs.Marshalizer = marshalizer
storageManagerArgs.Hasher = hasher
storageManagerArgs.MainStorer = trieStorer

trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions())
trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, testCommon.GetStorageManagerOptions())
argsAccCreator := factory.ArgsAccountCreator{
Hasher: hasher,
Marshaller: marshalizer,
Expand Down
19 changes: 11 additions & 8 deletions factory/api/apiResolverFactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,21 +594,24 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai
Identifier: dataRetriever.UserAccountsUnit.String(),
EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(),
StatsCollector: args.statusCoreComponents.StateStatsHandler(),
MaxSizeInMemory: args.generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory,
}
trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs)
if err != nil {
return nil, nil, err
}

argsAPIAccountsDB := state.ArgsAccountsDB{
Trie: merkleTrie,
Hasher: args.coreComponents.Hasher(),
Marshaller: args.coreComponents.InternalMarshalizer(),
AccountFactory: accountFactory,
StoragePruningManager: storagePruning,
AddressConverter: args.coreComponents.AddressPubKeyConverter(),
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
Trie: merkleTrie,
Hasher: args.coreComponents.Hasher(),
Marshaller: args.coreComponents.InternalMarshalizer(),
AccountFactory: accountFactory,
StoragePruningManager: storagePruning,
AddressConverter: args.coreComponents.AddressPubKeyConverter(),
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
// TODO check if this should be lower than in the processing adb
MaxDataTriesSizeInMemory: args.generalConfig.StateTriesConfig.DataTriesSizeInMemory,
}

provider, err := blockInfoProviders.NewCurrentBlockInfo(chainHandler)
Expand Down
11 changes: 9 additions & 2 deletions factory/api/apiResolverFactory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ import (
"github.com/stretchr/testify/require"
)

const unreachableStep = 10000
const (
unreachableStep = 10000
tenMBSize = uint64(10485760)
)

type failingSteps struct {
marshallerStepCounter int
Expand Down Expand Up @@ -305,7 +308,11 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs {
TrieStorageManagerConfig: config.TrieStorageManagerConfig{
SnapshotsGoroutineNum: 1,
},
StateTriesConfig: config.StateTriesConfig{},
StateTriesConfig: config.StateTriesConfig{
MaxUserTrieSizeInMemory: tenMBSize,
MaxPeerTrieSizeInMemory: tenMBSize,
DataTriesSizeInMemory: tenMBSize,
},
VirtualMachine: config.VirtualMachineServicesConfig{
Querying: config.QueryVirtualMachineConfig{
VirtualMachineConfig: config.VirtualMachineConfig{
Expand Down
28 changes: 15 additions & 13 deletions factory/processing/blockProcessorCreator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/multiversx/mx-chain-core-go/core"
"github.com/multiversx/mx-chain-core-go/hashing"
"github.com/multiversx/mx-chain-core-go/marshal"
common2 "github.com/multiversx/mx-chain-go/testscommon/common"
vmcommon "github.com/multiversx/mx-chain-vm-common-go"
"github.com/stretchr/testify/require"

Expand All @@ -26,7 +27,6 @@ import (
"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
"github.com/multiversx/mx-chain-go/testscommon/processMocks"
stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
storageManager "github.com/multiversx/mx-chain-go/testscommon/storage"
trieMock "github.com/multiversx/mx-chain-go/testscommon/trie"
"github.com/multiversx/mx-chain-go/trie"
)
Expand Down Expand Up @@ -91,13 +91,13 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) {
cryptoComponents := componentsMock.GetCryptoComponents(coreComponents)
networkComponents := componentsMock.GetNetworkComponents(cryptoComponents)

storageManagerArgs := storageManager.GetStorageManagerArgs()
storageManagerArgs := common2.GetStorageManagerArgs()
storageManagerArgs.Marshalizer = coreComponents.InternalMarshalizer()
storageManagerArgs.Hasher = coreComponents.Hasher()
storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions())
storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions())

storageManagerArgs.MainStorer = mock.NewMemDbMock()
storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions())
storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions())

trieStorageManagers := make(map[string]common.StorageManager)
trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser
Expand Down Expand Up @@ -209,20 +209,22 @@ func createAccountAdapter(
trieStorage common.StorageManager,
handler common.EnableEpochsHandler,
) (state.AccountsAdapter, error) {
tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler)
tenMbSize := uint64(10485760)
tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, tenMbSize)
if err != nil {
return nil, err
}

args := state.ArgsAccountsDB{
Trie: tr,
Hasher: hasher,
Marshaller: marshaller,
AccountFactory: accountFactory,
StoragePruningManager: disabled.NewDisabledStoragePruningManager(),
AddressConverter: &testscommon.PubkeyConverterMock{},
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
Trie: tr,
Hasher: hasher,
Marshaller: marshaller,
AccountFactory: accountFactory,
StoragePruningManager: disabled.NewDisabledStoragePruningManager(),
AddressConverter: &testscommon.PubkeyConverterMock{},
SnapshotsManager: disabledState.NewDisabledSnapshotsManager(),
StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
MaxDataTriesSizeInMemory: tenMbSize,
}
adb, err := state.NewAccountsDB(args)
if err != nil {
Expand Down
Loading
Loading