Merge pull request #6054 from multiversx/extend-overridable-configs-f…

…iles

Added more files in the overridable configs options and chain simulator fixes

cmd/node/config/prefs.toml

@@ -40,7 +40,8 @@
    # configuration of the node has the false value)
    # The Path indicates what value to change, while Value represents the new value in string format. The node operator must make sure
    # to follow the same type of the original value (ex: uint32: "37", float32: "37.0", bool: "true")
-   # File represents the file name that holds the configuration. Currently, the supported files are: config.toml, external.toml, p2p.toml and enableEpochs.toml
+   # File represents the file name that holds the configuration. Currently, the supported files are:
+   # api.toml, config.toml, economics.toml, enableEpochs.toml, enableRounds.toml, external.toml, fullArchiveP2P.toml, p2p.toml, ratings.toml, systemSmartContractsConfig.toml
    # -------------------------------
    # Un-comment and update the following section in order to enable config values overloading
    # -------------------------------

config/overridableConfig/configOverriding.go

@@ -10,33 +10,60 @@ import (
 )
 
 const (
+	apiTomlFile            = "api.toml"
 	configTomlFile         = "config.toml"
+	economicsTomlFile      = "economics.toml"
 	enableEpochsTomlFile   = "enableEpochs.toml"
-	p2pTomlFile            = "p2p.toml"
-	fullArchiveP2PTomlFile = "fullArchiveP2P.toml"
+	enableRoundsTomlFile   = "enableRounds.toml"
 	externalTomlFile       = "external.toml"
+	fullArchiveP2PTomlFile = "fullArchiveP2P.toml"
+	p2pTomlFile            = "p2p.toml"
+	ratingsTomlFile        = "ratings.toml"
+	systemSCTomlFile       = "systemSmartContractsConfig.toml"
 )
 
 var (
-	availableConfigFilesForOverriding = []string{configTomlFile, enableEpochsTomlFile, p2pTomlFile, externalTomlFile}
-	log                               = logger.GetOrCreate("config")
+	availableConfigFilesForOverriding = []string{
+		apiTomlFile,
+		configTomlFile,
+		economicsTomlFile,
+		enableEpochsTomlFile,
+		enableRoundsTomlFile,
+		externalTomlFile,
+		fullArchiveP2PTomlFile,
+		p2pTomlFile,
+		ratingsTomlFile,
+		systemSCTomlFile,
+	}
+	log = logger.GetOrCreate("config")
 )
 
 // OverrideConfigValues will override config values for the specified configurations
 func OverrideConfigValues(newConfigs []config.OverridableConfig, configs *config.Configs) error {
 	var err error
 	for _, newConfig := range newConfigs {
 		switch newConfig.File {
+		case apiTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.ApiRoutesConfig, newConfig.Path, newConfig.Value)
 		case configTomlFile:
 			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.GeneralConfig, newConfig.Path, newConfig.Value)
+		case economicsTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.EconomicsConfig, newConfig.Path, newConfig.Value)
 		case enableEpochsTomlFile:
 			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.EpochConfig, newConfig.Path, newConfig.Value)
-		case p2pTomlFile:
-			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.MainP2pConfig, newConfig.Path, newConfig.Value)
-		case fullArchiveP2PTomlFile:
-			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.FullArchiveP2pConfig, newConfig.Path, newConfig.Value)
+		case enableRoundsTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.RoundConfig, newConfig.Path, newConfig.Value)
 		case externalTomlFile:
 			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.ExternalConfig, newConfig.Path, newConfig.Value)
+		case fullArchiveP2PTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.FullArchiveP2pConfig, newConfig.Path, newConfig.Value)
+		case p2pTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.MainP2pConfig, newConfig.Path, newConfig.Value)
+		case ratingsTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.RatingsConfig, newConfig.Path, newConfig.Value)
+		case systemSCTomlFile:
+			err = reflectcommon.AdaptStructureValueBasedOnPath(configs.SystemSCConfig, newConfig.Path, newConfig.Value)
+
 		default:
 			err = fmt.Errorf("invalid config file <%s>. Available options are %s", newConfig.File, strings.Join(availableConfigFilesForOverriding, ","))
 		}

config/overridableConfig/configOverriding_test.go

@@ -22,7 +22,8 @@ func TestOverrideConfigValues(t *testing.T) {
 		t.Parallel()
 
 		err := OverrideConfigValues([]config.OverridableConfig{{File: "invalid.toml"}}, &config.Configs{})
-		require.Equal(t, "invalid config file <invalid.toml>. Available options are config.toml,enableEpochs.toml,p2p.toml,external.toml", err.Error())
+		availableOptionsString := "api.toml,config.toml,economics.toml,enableEpochs.toml,enableRounds.toml,external.toml,fullArchiveP2P.toml,p2p.toml,ratings.toml,systemSmartContractsConfig.toml"
+		require.Equal(t, "invalid config file <invalid.toml>. Available options are "+availableOptionsString, err.Error())
 	})
 
 	t.Run("nil config, should error", func(t *testing.T) {
@@ -81,4 +82,57 @@ func TestOverrideConfigValues(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, uint32(37), configs.EpochConfig.EnableEpochs.ESDTMetadataContinuousCleanupEnableEpoch)
 	})
+
+	t.Run("should work for api.toml", func(t *testing.T) {
+		t.Parallel()
+
+		configs := &config.Configs{ApiRoutesConfig: &config.ApiRoutesConfig{}}
+
+		err := OverrideConfigValues([]config.OverridableConfig{{Path: "Logging.LoggingEnabled", Value: "true", File: "api.toml"}}, configs)
+		require.NoError(t, err)
+		require.True(t, configs.ApiRoutesConfig.Logging.LoggingEnabled)
+	})
+
+	t.Run("should work for economics.toml", func(t *testing.T) {
+		t.Parallel()
+
+		configs := &config.Configs{EconomicsConfig: &config.EconomicsConfig{}}
+
+		err := OverrideConfigValues([]config.OverridableConfig{{Path: "GlobalSettings.GenesisTotalSupply", Value: "37", File: "economics.toml"}}, configs)
+		require.NoError(t, err)
+		require.Equal(t, "37", configs.EconomicsConfig.GlobalSettings.GenesisTotalSupply)
+	})
+
+	t.Run("should work for enableRounds.toml", func(t *testing.T) {
+		// TODO: fix this test
+		t.Skip("skipped, as this test requires the fix from this PR: https://github.com/multiversx/mx-chain-go/pull/5851")
+
+		t.Parallel()
+
+		configs := &config.Configs{RoundConfig: &config.RoundConfig{}}
+
+		err := OverrideConfigValues([]config.OverridableConfig{{Path: "RoundActivations.DisableAsyncCallV1.Round", Value: "37", File: "enableRounds.toml"}}, configs)
+		require.NoError(t, err)
+		require.Equal(t, uint32(37), configs.RoundConfig.RoundActivations["DisableAsyncCallV1"])
+	})
+
+	t.Run("should work for ratings.toml", func(t *testing.T) {
+		t.Parallel()
+
+		configs := &config.Configs{RatingsConfig: &config.RatingsConfig{}}
+
+		err := OverrideConfigValues([]config.OverridableConfig{{Path: "General.StartRating", Value: "37", File: "ratings.toml"}}, configs)
+		require.NoError(t, err)
+		require.Equal(t, uint32(37), configs.RatingsConfig.General.StartRating)
+	})
+
+	t.Run("should work for systemSmartContractsConfig.toml", func(t *testing.T) {
+		t.Parallel()
+
+		configs := &config.Configs{SystemSCConfig: &config.SystemSmartContractsConfig{}}
+
+		err := OverrideConfigValues([]config.OverridableConfig{{Path: "StakingSystemSCConfig.UnBondPeriod", Value: "37", File: "systemSmartContractsConfig.toml"}}, configs)
+		require.NoError(t, err)
+		require.Equal(t, uint64(37), configs.SystemSCConfig.StakingSystemSCConfig.UnBondPeriod)
+	})
 }

integrationTests/chainSimulator/interface.go

@@ -21,4 +21,5 @@ type ChainSimulator interface {
 	GenerateAndMintWalletAddress(targetShardID uint32, value *big.Int) (dtos.WalletAddress, error)
 	GetInitialWalletKeys() *dtos.InitialWalletKeys
 	GetAccount(address dtos.WalletAddress) (api.AccountResponse, error)
+	ForceResetValidatorStatisticsCache() error
 }

integrationTests/chainSimulator/staking/delegation_test.go

@@ -277,7 +277,7 @@ func testChainSimulatorMakeNewContractFromValidatorData(t *testing.T, cs chainSi
 	delegationAddressBech32 := metachainNode.GetCoreComponents().AddressPubKeyConverter().SilentEncode(delegationAddress, log)
 	log.Info("generated delegation address", "address", delegationAddressBech32)
 
-	err = metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err = cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 
 	testBLSKeyIsInQueueOrAuction(t, metachainNode, delegationAddress, blsKeys[0], addedStakedValue, 1)

integrationTests/chainSimulator/staking/jail_test.go

@@ -77,7 +77,7 @@ func testChainSimulatorJailAndUnJail(t *testing.T, targetEpoch int32, nodeStatus
 		AlterConfigsFunction: func(cfg *config.Configs) {
 			configs.SetStakingV4ActivationEpochs(cfg, stakingV4JailUnJailStep1EnableEpoch)
 			newNumNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake + 8 // 8 nodes until new nodes will be placed on queue
-			configs.SetMaxNumberOfNodesInConfigs(cfg, newNumNodes, numOfShards)
+			configs.SetMaxNumberOfNodesInConfigs(cfg, uint32(newNumNodes), 0, numOfShards)
 			configs.SetQuickJailRatingConfig(cfg)
 		},
 	})
@@ -179,7 +179,7 @@ func TestChainSimulator_FromQueueToAuctionList(t *testing.T) {
 			configs.SetQuickJailRatingConfig(cfg)
 
 			newNumNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake + 1
-			configs.SetMaxNumberOfNodesInConfigs(cfg, newNumNodes, numOfShards)
+			configs.SetMaxNumberOfNodesInConfigs(cfg, uint32(newNumNodes), 0, numOfShards)
 		},
 	})
 	require.Nil(t, err)
@@ -248,7 +248,7 @@ func TestChainSimulator_FromQueueToAuctionList(t *testing.T) {
 }
 
 func checkValidatorStatus(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator, blsKey string, expectedStatus string) {
-	err := cs.GetNodeHandler(core.MetachainShardId).GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err := cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 
 	validatorsStatistics, err := cs.GetNodeHandler(core.MetachainShardId).GetFacadeHandler().ValidatorStatisticsApi()

integrationTests/chainSimulator/staking/simpleStake_test.go

@@ -212,7 +212,7 @@ func TestChainSimulator_StakingV4Step2APICalls(t *testing.T) {
 	require.Nil(t, err)
 
 	// In step 1, only the previously staked node should be in auction list
-	err = metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err = cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 	auctionList, err := metachainNode.GetProcessComponents().ValidatorsProvider().GetAuctionList()
 	require.Nil(t, err)
@@ -229,7 +229,7 @@ func TestChainSimulator_StakingV4Step2APICalls(t *testing.T) {
 	require.Nil(t, err)
 
 	// after the re-stake process, the node should be in auction list
-	err = metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err = cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 	auctionList, err = metachainNode.GetProcessComponents().ValidatorsProvider().GetAuctionList()
 	require.Nil(t, err)

integrationTests/chainSimulator/staking/stakeAndUnStake_test.go

@@ -69,7 +69,7 @@ func TestChainSimulator_AddValidatorKey(t *testing.T) {
 		NumNodesWaitingListShard: 0,
 		AlterConfigsFunction: func(cfg *config.Configs) {
 			newNumNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake + 8 // 8 nodes until new nodes will be placed on queue
-			configs.SetMaxNumberOfNodesInConfigs(cfg, newNumNodes, numOfShards)
+			configs.SetMaxNumberOfNodesInConfigs(cfg, uint32(newNumNodes), 0, numOfShards)
 		},
 	})
 	require.Nil(t, err)
@@ -152,7 +152,7 @@ func TestChainSimulator_AddValidatorKey(t *testing.T) {
 	require.Nil(t, err)
 
 	metachainNode := cs.GetNodeHandler(core.MetachainShardId)
-	err = metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err = cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 	validatorStatistics, err := metachainNode.GetFacadeHandler().ValidatorStatisticsApi()
 	require.Nil(t, err)
@@ -200,8 +200,10 @@ func TestChainSimulator_AddANewValidatorAfterStakingV4(t *testing.T) {
 		AlterConfigsFunction: func(cfg *config.Configs) {
 			cfg.SystemSCConfig.StakingSystemSCConfig.NodeLimitPercentage = 1
 			cfg.GeneralConfig.ValidatorStatistics.CacheRefreshIntervalInSec = 1
-			newNumNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake + 8 // 8 nodes until new nodes will be placed on queue
-			configs.SetMaxNumberOfNodesInConfigs(cfg, newNumNodes, numOfShards)
+			eligibleNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake
+			// 8 nodes until new nodes will be placed on queue
+			waitingNodes := uint32(8)
+			configs.SetMaxNumberOfNodesInConfigs(cfg, uint32(eligibleNodes), waitingNodes, numOfShards)
 		},
 	})
 	require.Nil(t, err)
@@ -262,7 +264,7 @@ func TestChainSimulator_AddANewValidatorAfterStakingV4(t *testing.T) {
 	require.Nil(t, err)
 
 	metachainNode := cs.GetNodeHandler(core.MetachainShardId)
-	err = metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+	err = cs.ForceResetValidatorStatisticsCache()
 	require.Nil(t, err)
 	results, err := metachainNode.GetFacadeHandler().AuctionListApi()
 	require.Nil(t, err)
@@ -328,7 +330,7 @@ func testStakeUnStakeUnBond(t *testing.T, targetEpoch int32) {
 			cfg.SystemSCConfig.StakingSystemSCConfig.UnBondPeriod = 1
 			cfg.SystemSCConfig.StakingSystemSCConfig.UnBondPeriodInEpochs = 1
 			newNumNodes := cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake + 10
-			configs.SetMaxNumberOfNodesInConfigs(cfg, newNumNodes, numOfShards)
+			configs.SetMaxNumberOfNodesInConfigs(cfg, uint32(newNumNodes), 0, numOfShards)
 		},
 	})
 	require.Nil(t, err)

node/chainSimulator/chainSimulator.go

@@ -212,6 +212,16 @@ func (s *simulator) GenerateBlocksUntilEpochIsReached(targetEpoch int32) error {
 	return fmt.Errorf("exceeded rounds to generate blocks")
 }
 
+// ForceResetValidatorStatisticsCache will force the reset of the cache used for the validators statistics endpoint
+func (s *simulator) ForceResetValidatorStatisticsCache() error {
+	metachainNode := s.GetNodeHandler(core.MetachainShardId)
+	if check.IfNil(metachainNode) {
+		return errNilMetachainNode
+	}
+
+	return metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
+}
+
 func (s *simulator) isTargetEpochReached(targetEpoch int32) (bool, error) {
 	metachainNode := s.nodes[core.MetachainShardId]
 	metachainEpoch := metachainNode.GetCoreComponents().EnableEpochsHandler().GetCurrentEpoch()

node/chainSimulator/configs/configs.go

@@ -103,10 +103,10 @@ func CreateChainSimulatorConfigs(args ArgsChainSimulatorConfigs) (*ArgsConfigsSi
 	configs.GeneralConfig.SmartContractsStorageForSCQuery.DB.Type = string(storageunit.MemoryDB)
 	configs.GeneralConfig.SmartContractsStorageSimulate.DB.Type = string(storageunit.MemoryDB)
 
-	maxNumNodes := uint64((args.MinNodesPerShard+args.NumNodesWaitingListShard)*args.NumOfShards) +
-		uint64(args.MetaChainMinNodes+args.NumNodesWaitingListMeta)
+	eligibleNodes := args.MinNodesPerShard*args.NumOfShards + args.MetaChainMinNodes
+	waitingNodes := args.NumNodesWaitingListShard*args.NumOfShards + args.NumNodesWaitingListMeta
 
-	SetMaxNumberOfNodesInConfigs(configs, maxNumNodes, args.NumOfShards)
+	SetMaxNumberOfNodesInConfigs(configs, eligibleNodes, waitingNodes, args.NumOfShards)
 
 	// set compatible trie configs
 	configs.GeneralConfig.StateTriesConfig.SnapshotsEnabled = false
@@ -141,17 +141,23 @@ func CreateChainSimulatorConfigs(args ArgsChainSimulatorConfigs) (*ArgsConfigsSi
 }
 
 // SetMaxNumberOfNodesInConfigs will correctly set the max number of nodes in configs
-func SetMaxNumberOfNodesInConfigs(cfg *config.Configs, maxNumNodes uint64, numOfShards uint32) {
-	cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake = maxNumNodes
+func SetMaxNumberOfNodesInConfigs(cfg *config.Configs, eligibleNodes uint32, waitingNodes uint32, numOfShards uint32) {
+	cfg.SystemSCConfig.StakingSystemSCConfig.MaxNumberOfNodesForStake = uint64(eligibleNodes + waitingNodes)
 	numMaxNumNodesEnableEpochs := len(cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch)
 	for idx := 0; idx < numMaxNumNodesEnableEpochs-1; idx++ {
-		cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[idx].MaxNumNodes = uint32(maxNumNodes)
+		cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[idx].MaxNumNodes = eligibleNodes + waitingNodes
 	}
 
 	cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[numMaxNumNodesEnableEpochs-1].EpochEnable = cfg.EpochConfig.EnableEpochs.StakingV4Step3EnableEpoch
 	prevEntry := cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[numMaxNumNodesEnableEpochs-2]
 	cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[numMaxNumNodesEnableEpochs-1].NodesToShufflePerShard = prevEntry.NodesToShufflePerShard
-	cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[numMaxNumNodesEnableEpochs-1].MaxNumNodes = prevEntry.MaxNumNodes - (numOfShards+1)*prevEntry.NodesToShufflePerShard
+
+	stakingV4NumNodes := eligibleNodes + waitingNodes
+	if stakingV4NumNodes-(numOfShards+1)*prevEntry.NodesToShufflePerShard >= eligibleNodes {
+		// prevent the case in which we are decreasing the eligible number of nodes because we are working with 0 waiting list size
+		stakingV4NumNodes -= (numOfShards + 1) * prevEntry.NodesToShufflePerShard
+	}
+	cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[numMaxNumNodesEnableEpochs-1].MaxNumNodes = stakingV4NumNodes
 }
 
 // SetQuickJailRatingConfig will set the rating config in a way that leads to rapid jailing of a node