AUTO-10236: add integration tests for max gas price check (#12974)

* AUTO-10214: compare max gas price with current gas price in simulation process

* refactor and add tests

* linting (#12960)

* linting * 2

* fix linting

* AUTO-10236

* fix go mod

* update test json

* improve max gas price integration tests

* AUTO-10214: compare max gas price with current gas price in simulation process

* refactor and add tests

* linting (#12960)

* linting * 2

* fix linting

* create opts with latest block

* add some logs

* fix bug and update logs

* update

* update

.changeset/witty-weeks-kneel.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#added an integration test for max gas price check

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/encoding/interface.go

@@ -26,15 +26,12 @@ const (
 	UpkeepFailureReasonRegistryPaused          UpkeepFailureReason = 9
 	// leaving a gap here for more onchain failure reasons in the future
 	// upkeep failure offchain reasons
-	UpkeepFailureReasonMercuryAccessNotAllowed      UpkeepFailureReason = 32
-	UpkeepFailureReasonTxHashNoLongerExists         UpkeepFailureReason = 33
-	UpkeepFailureReasonInvalidRevertDataInput       UpkeepFailureReason = 34
-	UpkeepFailureReasonSimulationFailed             UpkeepFailureReason = 35
-	UpkeepFailureReasonTxHashReorged                UpkeepFailureReason = 36
-	UpkeepFailureReasonFailToRetrieveOffchainConfig UpkeepFailureReason = 37
-	UpkeepFailureReasonFailToParseOffchainConfig    UpkeepFailureReason = 38
-	UpkeepFailureReasonFailToRetrieveGasPrice       UpkeepFailureReason = 39
-	UpkeepFailureReasonGasPriceTooHigh              UpkeepFailureReason = 40
+	UpkeepFailureReasonMercuryAccessNotAllowed UpkeepFailureReason = 32
+	UpkeepFailureReasonTxHashNoLongerExists    UpkeepFailureReason = 33
+	UpkeepFailureReasonInvalidRevertDataInput  UpkeepFailureReason = 34
+	UpkeepFailureReasonSimulationFailed        UpkeepFailureReason = 35
+	UpkeepFailureReasonTxHashReorged           UpkeepFailureReason = 36
+	UpkeepFailureReasonGasPriceTooHigh         UpkeepFailureReason = 37
 
 	// pipeline execution error
 	NoPipelineError               PipelineExecutionState = 0

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice.go

@@ -24,45 +24,47 @@ type UpkeepOffchainConfig struct {
 	MaxGasPrice *big.Int `json:"maxGasPrice" cbor:"maxGasPrice"`
 }
 
+// CheckGasPrice retrieves the current gas price and compare against the max gas price configured in upkeep's offchain config
+// any errors in offchain config decoding will result in max gas price check disabled
 func CheckGasPrice(ctx context.Context, upkeepId *big.Int, oc []byte, ge gas.EvmFeeEstimator, lggr logger.Logger) encoding.UpkeepFailureReason {
 	if len(oc) == 0 {
 		return encoding.UpkeepFailureReasonNone
 	}
 
 	var offchainConfig UpkeepOffchainConfig
 	if err := cbor.ParseDietCBORToStruct(oc, &offchainConfig); err != nil {
-		lggr.Errorw("failed to parse upkeep offchain config", "upkeepId", upkeepId.String(), "err", err)
-		return encoding.UpkeepFailureReasonFailToParseOffchainConfig
+		lggr.Errorw("failed to parse upkeep offchain config, gas price check is disabled", "upkeepId", upkeepId.String(), "err", err)
+		return encoding.UpkeepFailureReasonNone
 	}
 	if offchainConfig.MaxGasPrice == nil {
-		lggr.Infow("maxGasPrice is not configured in upkeep offchain config", "upkeepId", upkeepId.String())
+		lggr.Infow("maxGasPrice is not configured in upkeep offchain config, gas price check is disabled", "upkeepId", upkeepId.String())
 		return encoding.UpkeepFailureReasonNone
 	}
 	lggr.Infof("successfully decode offchain config for %s", upkeepId.String())
 	lggr.Infof("max gas price for %s is %s", upkeepId.String(), offchainConfig.MaxGasPrice.String())
 
 	fee, _, err := ge.GetFee(ctx, []byte{}, feeLimit, assets.NewWei(big.NewInt(maxFeePrice)))
 	if err != nil {
-		lggr.Errorw("failed to get fee", "upkeepId", upkeepId.String(), "err", err)
-		return encoding.UpkeepFailureReasonFailToRetrieveGasPrice
+		lggr.Errorw("failed to get fee, gas price check is disabled", "upkeepId", upkeepId.String(), "err", err)
+		return encoding.UpkeepFailureReasonNone
 	}
 
 	if fee.ValidDynamic() {
 		lggr.Infof("current gas price EIP-1559 is fee cap %s, tip cap %s", fee.DynamicFeeCap.String(), fee.DynamicTipCap.String())
 		if fee.DynamicFeeCap.Cmp(assets.NewWei(offchainConfig.MaxGasPrice)) > 0 {
 			// current gas price is higher than max gas price
-			lggr.Warnf("max gas price %s for %s is LOWER than current gas price %s", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.DynamicFeeCap.String())
+			lggr.Warnf("maxGasPrice %s for %s is LOWER than current gas price %d", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.DynamicFeeCap.Int64())
 			return encoding.UpkeepFailureReasonGasPriceTooHigh
 		}
-		lggr.Infof("max gas price %s for %s is HIGHER than current gas price %s", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.DynamicFeeCap.String())
+		lggr.Infof("maxGasPrice %s for %s is HIGHER than current gas price %d", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.DynamicFeeCap.Int64())
 	} else {
 		lggr.Infof("current gas price legacy is %s", fee.Legacy.String())
 		if fee.Legacy.Cmp(assets.NewWei(offchainConfig.MaxGasPrice)) > 0 {
 			// current gas price is higher than max gas price
-			lggr.Infof("max gas price %s for %s is LOWER than current gas price %s", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.Legacy.String())
+			lggr.Infof("maxGasPrice %s for %s is LOWER than current gas price %d", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.Legacy.Int64())
 			return encoding.UpkeepFailureReasonGasPriceTooHigh
 		}
-		lggr.Infof("max gas price %s for %s is HIGHER than current gas price %s", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.Legacy.String())
+		lggr.Infof("maxGasPrice %s for %s is HIGHER than current gas price %d", offchainConfig.MaxGasPrice.String(), upkeepId.String(), fee.Legacy.Int64())
 	}
 
 	return encoding.UpkeepFailureReasonNone

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice/gasprice_test.go

@@ -31,6 +31,7 @@ func TestGasPrice_Check(t *testing.T) {
 		CurrentLegacyGasPrice  *big.Int
 		CurrentDynamicGasPrice *big.Int
 		ExpectedResult         encoding.UpkeepFailureReason
+		FailedToGetFee         bool
 		NotConfigured          bool
 		ParsingFailed          bool
 	}{
@@ -47,12 +48,13 @@ func TestGasPrice_Check(t *testing.T) {
 			Name:           "fail to parse offchain config",
 			ParsingFailed:  true,
 			MaxGasPrice:    big.NewInt(10_000_000_000),
-			ExpectedResult: encoding.UpkeepFailureReasonFailToParseOffchainConfig,
+			ExpectedResult: encoding.UpkeepFailureReasonNone,
 		},
 		{
 			Name:           "fail to retrieve current gas price",
 			MaxGasPrice:    big.NewInt(8_000_000_000),
-			ExpectedResult: encoding.UpkeepFailureReasonFailToRetrieveGasPrice,
+			FailedToGetFee: true,
+			ExpectedResult: encoding.UpkeepFailureReasonNone,
 		},
 		{
 			Name:                  "current gas price is too high - legacy",
@@ -83,7 +85,7 @@ func TestGasPrice_Check(t *testing.T) {
 		t.Run(test.Name, func(t *testing.T) {
 			ctx := testutils.Context(t)
 			ge := gasMocks.NewEvmFeeEstimator(t)
-			if test.ExpectedResult == encoding.UpkeepFailureReasonFailToRetrieveGasPrice {
+			if test.FailedToGetFee {
 				ge.On("GetFee", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
 					gas.EvmFee{},
 					feeLimit,

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry.go

@@ -635,5 +635,8 @@ func (r *EvmRegistry) fetchTriggerConfig(id *big.Int) ([]byte, error) {
 func (r *EvmRegistry) fetchUpkeepOffchainConfig(id *big.Int) ([]byte, error) {
 	opts := r.buildCallOpts(r.ctx, nil)
 	ui, err := r.registry.GetUpkeep(opts, id)
-	return ui.OffchainConfig, err
+	if err != nil {
+		return []byte{}, err
+	}
+	return ui.OffchainConfig, nil
 }

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/registry_check_pipeline.go

@@ -31,10 +31,6 @@ type checkResult struct {
 	err error
 }
 
-type UpkeepOffchainConfig struct {
-	MaxGasPrice *big.Int `json:"maxGasPrice" cbor:"maxGasPrice"`
-}
-
 func (r *EvmRegistry) CheckUpkeeps(ctx context.Context, keys ...ocr2keepers.UpkeepPayload) ([]ocr2keepers.CheckResult, error) {
 	r.lggr.Debugw("Checking upkeeps", "upkeeps", keys)
 	for i := range keys {
@@ -312,15 +308,15 @@ func (r *EvmRegistry) simulatePerformUpkeeps(ctx context.Context, checkResults [
 
 		oc, err := r.fetchUpkeepOffchainConfig(upkeepId)
 		if err != nil {
-			r.lggr.Warnw("failed get offchain config", "err", err, "upkeepId", upkeepId, "block", block)
-			checkResults[i].Eligible = false
-			checkResults[i].PipelineExecutionState = uint8(encoding.UpkeepFailureReasonFailToRetrieveOffchainConfig)
-			continue
+			// this is mostly caused by RPC flakiness
+			r.lggr.Errorw("failed get offchain config, gas price check will be disabled", "err", err, "upkeepId", upkeepId, "block", block)
 		}
 		fr := gasprice.CheckGasPrice(ctx, upkeepId, oc, r.ge, r.lggr)
-		if fr != encoding.UpkeepFailureReasonNone {
+		if uint8(fr) == uint8(encoding.UpkeepFailureReasonGasPriceTooHigh) {
+			r.lggr.Infof("upkeep %s upkeep failure reason is %d", upkeepId, fr)
 			checkResults[i].Eligible = false
-			checkResults[i].PipelineExecutionState = uint8(fr)
+			checkResults[i].Retryable = false
+			checkResults[i].IneligibilityReason = uint8(fr)
 			continue
 		}
 

integration-tests/contracts/ethereum_keeper_contracts.go

@@ -87,6 +87,7 @@ type KeeperRegistry interface {
 	UpdateCheckData(id *big.Int, newCheckData []byte) error
 	SetUpkeepTriggerConfig(id *big.Int, triggerConfig []byte) error
 	SetUpkeepPrivilegeConfig(id *big.Int, privilegeConfig []byte) error
+	SetUpkeepOffchainConfig(id *big.Int, offchainConfig []byte) error
 	RegistryOwnerAddress() common.Address
 	ChainModuleAddress() common.Address
 	ReorgProtectionEnabled() bool
@@ -1225,6 +1226,46 @@ func (v *EthereumKeeperRegistry) UnpauseUpkeep(id *big.Int) error {
 	}
 }
 
+func (v *EthereumKeeperRegistry) SetUpkeepOffchainConfig(id *big.Int, offchainConfig []byte) error {
+	switch v.version {
+	case ethereum.RegistryVersion_2_0:
+		opts, err := v.client.TransactionOpts(v.client.GetDefaultWallet())
+		if err != nil {
+			return err
+		}
+
+		tx, err := v.registry2_0.SetUpkeepOffchainConfig(opts, id, offchainConfig)
+		if err != nil {
+			return err
+		}
+		return v.client.ProcessTransaction(tx)
+	case ethereum.RegistryVersion_2_1:
+		opts, err := v.client.TransactionOpts(v.client.GetDefaultWallet())
+		if err != nil {
+			return err
+		}
+
+		tx, err := v.registry2_1.SetUpkeepOffchainConfig(opts, id, offchainConfig)
+		if err != nil {
+			return err
+		}
+		return v.client.ProcessTransaction(tx)
+	case ethereum.RegistryVersion_2_2:
+		opts, err := v.client.TransactionOpts(v.client.GetDefaultWallet())
+		if err != nil {
+			return err
+		}
+
+		tx, err := v.registry2_2.SetUpkeepOffchainConfig(opts, id, offchainConfig)
+		if err != nil {
+			return err
+		}
+		return v.client.ProcessTransaction(tx)
+	default:
+		return fmt.Errorf("SetUpkeepOffchainConfig is not supported by keeper registry version %d", v.version)
+	}
+}
+
 // Parses upkeep performed log
 func (v *EthereumKeeperRegistry) ParseUpkeepPerformedLog(log *types.Log) (*UpkeepPerformedLog, error) {
 	switch v.version {

integration-tests/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/cli/go-gh/v2 v2.0.0
 	github.com/ethereum/go-ethereum v1.13.8
+	github.com/fxamacker/cbor/v2 v2.5.0
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
@@ -174,7 +175,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/fxamacker/cbor/v2 v2.5.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gagliardetto/binary v0.7.7 // indirect
 	github.com/gagliardetto/solana-go v1.8.4 // indirect

integration-tests/smoke/automation_test.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/fxamacker/cbor/v2"
 	"github.com/onsi/gomega"
 	"github.com/stretchr/testify/require"
 
@@ -34,6 +36,7 @@ import (
 	"github.com/smartcontractkit/chainlink/integration-tests/types/config/node"
 	ac "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_compatible_utils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/gasprice"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/mercury/streams"
 )
 
@@ -190,7 +193,7 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool, automationTestConfig t
 						for i := 0; i < len(upkeepIDs); i++ {
 							counter, err := consumers[i].Counter(testcontext.Get(t))
 							require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
-							l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep ID", i).Msg("Number of upkeeps performed")
+							l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep index", i).Msg("Number of upkeeps performed")
 							g.Expect(counter.Int64()).Should(gomega.BeNumerically(">=", int64(expect)),
 								"Expected consumer counter to be greater than %d, but got %d", expect, counter.Int64())
 						}
@@ -631,7 +634,7 @@ func TestAutomationRegisterUpkeep(t *testing.T) {
 						"Expected consumer counter to be greater than 0, but got %d", counter.Int64())
 					l.Info().
 						Int64("Upkeep counter", counter.Int64()).
-						Int64("Upkeep ID", int64(i)).
+						Int64("Upkeep index", int64(i)).
 						Msg("Number of upkeeps performed")
 				}
 			}, "4m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~1m for performing each upkeep once, ~2m buffer
@@ -657,7 +660,7 @@ func TestAutomationRegisterUpkeep(t *testing.T) {
 					g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Calling consumer's counter shouldn't fail")
 
 					l.Info().
-						Int64("Upkeep ID", int64(i)).
+						Int64("Upkeep index", int64(i)).
 						Int64("Upkeep counter", currentCounter.Int64()).
 						Int64("initial counter", initialCounters[i].Int64()).
 						Msg("Number of upkeeps performed")
@@ -1120,6 +1123,117 @@ func TestUpdateCheckData(t *testing.T) {
 	}
 }
 
+func TestSetOffchainConfigWithMaxGasPrice(t *testing.T) {
+	t.Parallel()
+	registryVersions := map[string]ethereum.KeeperRegistryVersion{
+		// registry20 also has upkeep offchain config but the max gas price check is not implemented
+		"registry_2_1": ethereum.RegistryVersion_2_1,
+		"registry_2_2": ethereum.RegistryVersion_2_2,
+	}
+
+	for n, rv := range registryVersions {
+		name := n
+		registryVersion := rv
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			l := logging.GetTestLogger(t)
+			config, err := tc.GetConfig("Smoke", tc.Automation)
+			if err != nil {
+				t.Fatal(err)
+			}
+			a := setupAutomationTestDocker(
+				t, registryVersion, automationDefaultRegistryConfig(config), false, false, &config,
+			)
+
+			consumers, upkeepIDs := actions.DeployConsumers(
+				t,
+				a.Registry,
+				a.Registrar,
+				a.LinkToken,
+				a.Deployer,
+				a.ChainClient,
+				defaultAmountOfUpkeeps,
+				big.NewInt(automationDefaultLinkFunds),
+				automationDefaultUpkeepGasLimit,
+				false,
+				false,
+			)
+			gom := gomega.NewGomegaWithT(t)
+
+			l.Info().Msg("waiting for all upkeeps to be performed at least once")
+			gom.Eventually(func(g gomega.Gomega) {
+				for i := 0; i < len(upkeepIDs); i++ {
+					counter, err := consumers[i].Counter(testcontext.Get(t))
+					g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index %d", i)
+					g.Expect(counter.Int64()).Should(gomega.BeNumerically(">", int64(0)),
+						"Expected consumer counter to be greater than 0, but got %d")
+				}
+			}, "3m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~1m for performing each upkeep once, ~2m buffer
+
+			// set the maxGasPrice to 1 wei
+			uoc, _ := cbor.Marshal(gasprice.UpkeepOffchainConfig{MaxGasPrice: big.NewInt(1)})
+			l.Info().Msgf("setting all upkeeps' offchain config to %s, which means maxGasPrice is 1 wei", hexutil.Encode(uoc))
+			for _, uid := range upkeepIDs {
+				err = a.Registry.SetUpkeepOffchainConfig(uid, uoc)
+				require.NoError(t, err, "Error setting upkeep offchain config")
+				err = a.ChainClient.WaitForEvents()
+				require.NoError(t, err, "Error waiting for events from setting upkeep offchain config")
+			}
+
+			// Store how many times each upkeep performed once their offchain config is set with maxGasPrice = 1 wei
+			var countersAfterSettingLowMaxGasPrice = make([]*big.Int, len(upkeepIDs))
+			for i := 0; i < len(upkeepIDs); i++ {
+				countersAfterSettingLowMaxGasPrice[i], err = consumers[i].Counter(testcontext.Get(t))
+				require.NoError(t, err, "Failed to retrieve consumer counter for upkeep at index %d", i)
+				l.Info().Int64("Upkeep Performed times", countersAfterSettingLowMaxGasPrice[i].Int64()).Int("Upkeep index", i).Msg("Number of upkeeps performed")
+			}
+
+			var latestCounter *big.Int
+			// the counters of all the upkeeps should stay constant because they are no longer getting serviced
+			gom.Consistently(func(g gomega.Gomega) {
+				for i := 0; i < len(upkeepIDs); i++ {
+					latestCounter, err = consumers[i].Counter(testcontext.Get(t))
+					g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index %d", i)
+					g.Expect(latestCounter.Int64()).Should(gomega.Equal(countersAfterSettingLowMaxGasPrice[i].Int64()),
+						"Expected consumer counter to remain constant at %d, but got %d",
+						countersAfterSettingLowMaxGasPrice[i].Int64(), latestCounter.Int64())
+				}
+			}, "2m", "1s").Should(gomega.Succeed())
+			l.Info().Msg("no upkeeps is performed because their max gas price is only 1 wei")
+
+			// setting offchain config with a high max gas price for the first upkeep, it should perform again while
+			// other upkeeps should not perform
+			// set the maxGasPrice to 500 gwei for the first upkeep
+			uoc, _ = cbor.Marshal(gasprice.UpkeepOffchainConfig{MaxGasPrice: big.NewInt(500_000_000_000)})
+			l.Info().Msgf("setting the first upkeeps' offchain config to %s, which means maxGasPrice is 500 gwei", hexutil.Encode(uoc))
+			err = a.Registry.SetUpkeepOffchainConfig(upkeepIDs[0], uoc)
+			require.NoError(t, err, "Error setting upkeep offchain config")
+
+			// the counters of all other upkeeps should stay constant because their max gas price remains very low
+			gom.Consistently(func(g gomega.Gomega) {
+				for i := 1; i < len(upkeepIDs); i++ {
+					latestCounter, err = consumers[i].Counter(testcontext.Get(t))
+					g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index %d", i)
+					g.Expect(latestCounter.Int64()).Should(gomega.Equal(countersAfterSettingLowMaxGasPrice[i].Int64()),
+						"Expected consumer counter to remain constant at %d, but got %d",
+						countersAfterSettingLowMaxGasPrice[i].Int64(), latestCounter.Int64())
+				}
+			}, "2m", "1s").Should(gomega.Succeed())
+			l.Info().Msg("all the rest upkeeps did not perform again because their max gas price remains 1 wei")
+
+			// the first upkeep should start performing again
+			gom.Eventually(func(g gomega.Gomega) {
+				latestCounter, err = consumers[0].Counter(testcontext.Get(t))
+				g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index 0")
+				g.Expect(latestCounter.Int64()).Should(gomega.BeNumerically(">", countersAfterSettingLowMaxGasPrice[0].Int64()),
+					"Expected consumer counter to be greater than %d, but got %d",
+					countersAfterSettingLowMaxGasPrice[0].Int64(), latestCounter.Int64())
+			}, "2m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~1m for performing each upkeep once, ~2m buffer
+			l.Info().Int64("Upkeep Performed times", latestCounter.Int64()).Msg("the first upkeep performed again")
+		})
+	}
+}
+
 func setupAutomationTestDocker(
 	t *testing.T,
 	registryVersion ethereum.KeeperRegistryVersion,