Merge branch 'develop' into unittests

.changeset/brave-dots-breathe.md

@@ -0,0 +1,7 @@
+---
+"chainlink": minor
+---
+
+Added a new CLI command, `blocks find-lca,` which finds the latest block that is available in both the database and on the chain for the specified chain.
+Added a new CLI command, `node remove-blocks,` which removes all blocks and logs greater than or equal to the specified block number.
+#nops #added

.changeset/fresh-moles-explode.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+core/chains/evm/logpoller: Stricter finality checks in LogPoller, to be more robust during rpc failover events #updated

.changeset/pink-schools-provide.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#bugfix Fixed an issue where the `rebroadcast-transactions` commands did not execute config validation.

.changeset/quick-fishes-heal.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+#changed
+Added prefix `RPCClient returned error ({RPC_NAME})` to RPC errors to simplify filtering of RPC related issues.

.changeset/smooth-comics-love.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+update keystone gethwrapper with remove operator function #internal

.changeset/sour-jars-cross.md

@@ -0,0 +1,13 @@
+---
+"chainlink": patch
+---
+
+#added
+
+Add configurability to mercury transmitter
+
+```toml
+[Mercury.Transmitter]
+TransmitQueueMaxSize = 10_000 # Default
+TransmitTimeout = "5s" # Default
+```

.changeset/tidy-trees-tie.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#changed Updating the log trigger log provider's readMaxBatchSize to 56

.github/actions/setup-create-base64-upgrade-config/action.yml

@@ -92,6 +92,7 @@ runs:
         [ChainlinkUpgradeImage]
         image="$UPGRADE_IMAGE"
         version="$UPGRADE_VERSION"
+        postgres_version="$CHAINLINK_POSTGRES_VERSION"
 
         [Logging]
         test_log_collect=$test_log_collect

.github/workflows/integration-tests.yml

@@ -218,40 +218,6 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           dep_evm_sha: ${{ inputs.evm-ref }}
 
-  build-test-image:
-    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
-    environment: integration
-    permissions:
-      id-token: write
-      contents: read
-    name: Build Test Image
-    runs-on: ubuntu22.04-16cores-64GB
-    needs: [changes]
-    steps:
-      - name: Collect Metrics
-        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ${{ env.COLLECTION_ID }}-build-test-image
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Build Test Image
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-        with:
-          repository: smartcontractkit/chainlink
-          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
-      - name: Build Test Image
-        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: ./.github/actions/build-test-image
-        with:
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-
   compare-tests:
     needs: [changes]
     runs-on: ubuntu-latest
@@ -726,7 +692,7 @@ jobs:
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          QA_KUBECONFIG: ""
           should_tidy: "false"
       - name: Show Otel-Collector Logs
         if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
@@ -830,6 +796,7 @@ jobs:
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
   eth-smoke-go-mod-cache:
+
     environment: integration
     needs: [eth-smoke-tests]
     runs-on: ubuntu-latest
@@ -863,7 +830,7 @@ jobs:
       id-token: write
       contents: read
     runs-on: ubuntu-latest
-    needs: [build-chainlink, changes, build-test-image]
+    needs: [build-chainlink, changes]
     # Only run migration tests on new tags
     if: startsWith(github.ref, 'refs/tags/')
     env:
@@ -876,6 +843,17 @@ jobs:
       TEST_LOG_LEVEL: debug
       TEST_SUITE: migration
     steps:
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
+        with:
+          id: ${{ env.COLLECTION_ID }}-migration-tests
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Version Migration Tests
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
       - name: Checkout the repo
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
@@ -886,7 +864,12 @@ jobs:
         run: |
           untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name)
           latest_version="${untrimmed_ver:1}"
-          echo "latest_version=${latest_version} | tee -a $GITHUB_OUTPUT"
+          # Check if latest_version is empty
+          if [ -z "$latest_version" ]; then
+              echo "Error: The latest_version is empty. The migration tests need a verison to run."
+              exit 1
+          fi
+          echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT"
       - name: Name Versions
         run: |
           echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'"
@@ -898,13 +881,22 @@ jobs:
           chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }}
           upgradeImage: ${{ env.UPGRADE_IMAGE }}
           upgradeVersion: ${{ env.UPGRADE_VERSION }}
+          runId: ${{ github.run_id }}
+          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
+          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: ${{ vars.GRAFANA_URL }}
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Migration Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
+          test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
+          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: node-migration-test-logs
           artifacts_location: |
             ./integration-tests/migration/logs
@@ -916,28 +908,24 @@ jobs:
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          QA_KUBECONFIG: ""
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
+          should_tidy: "false"
       - name: Upload Coverage Data
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-migration-tests
           path: .covdata
           retention-days: 1          
-
-      - name: Collect Metrics
-        if: always()
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
+      - name: Notify Slack
+        if: failure() && github.event_name != 'workflow_dispatch'
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
         with:
-          id: ${{ env.COLLECTION_ID }}-migration-tests
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Version Migration Tests
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
+          channel-id: "#team-test-tooling-internal"
+          slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: ${{ job.html_url }}\n${{ format('https://github.com/smartcontractkit/chainlink/actions/runs/{0}', github.run_id) }}"
 
   ## Solana Section
   get_solana_sha:

common/client/models.go

@@ -28,6 +28,35 @@ var sendTxSevereErrors = []SendTxReturnCode{Fatal, Underpriced, Unsupported, Exc
 // sendTxSuccessfulCodes - error codes which signal that transaction was accepted by the node
 var sendTxSuccessfulCodes = []SendTxReturnCode{Successful, TransactionAlreadyKnown}
 
+func (c SendTxReturnCode) String() string {
+	switch c {
+	case Successful:
+		return "Successful"
+	case Fatal:
+		return "Fatal"
+	case Retryable:
+		return "Retryable"
+	case Underpriced:
+		return "Underpriced"
+	case Unknown:
+		return "Unknown"
+	case Unsupported:
+		return "Unsupported"
+	case TransactionAlreadyKnown:
+		return "TransactionAlreadyKnown"
+	case InsufficientFunds:
+		return "InsufficientFunds"
+	case ExceedsMaxFee:
+		return "ExceedsMaxFee"
+	case FeeOutOfValidRange:
+		return "FeeOutOfValidRange"
+	case OutOfCounters:
+		return "OutOfCounters"
+	default:
+		return fmt.Sprintf("SendTxReturnCode(%d)", c)
+	}
+}
+
 type NodeTier int
 
 const (

common/client/models_test.go

@@ -0,0 +1,16 @@
+package client
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSendTxReturnCode_String(t *testing.T) {
+	// ensure all the SendTxReturnCodes have proper name
+	for c := 1; c < int(sendTxReturnCodeLen); c++ {
+		strC := SendTxReturnCode(c).String()
+		if strings.Contains(strC, "SendTxReturnCode(") {
+			t.Errorf("Expected %s to have a proper string representation", strC)
+		}
+	}
+}

common/client/multi_node.go

@@ -561,6 +561,13 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	return n.RPC().PendingSequenceAt(ctx, addr)
 }
 
+type sendTxErrors map[SendTxReturnCode][]error
+
+// String - returns string representation of the errors map. Required by logger to properly represent the value
+func (errs sendTxErrors) String() string {
+	return fmt.Sprint(map[SendTxReturnCode][]error(errs))
+}
+
 func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM]) SendEmptyTransaction(
 	ctx context.Context,
 	newTxAttempt func(seq SEQ, feeLimit uint32, fee FEE, fromAddress ADDR) (attempt any, err error),
@@ -602,7 +609,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	ctx, cancel := c.chStop.Ctx(ctx)
 	defer cancel()
 	requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum))
-	errorsByCode := map[SendTxReturnCode][]error{}
+	errorsByCode := sendTxErrors{}
 	var softTimeoutChan <-chan time.Time
 	var resultsCount int
 loop:
@@ -639,7 +646,7 @@ loop:
 
 func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult) {
 	defer c.wg.Done()
-	resultsByCode := map[SendTxReturnCode][]error{}
+	resultsByCode := sendTxErrors{}
 	// txResults eventually will be closed
 	for txResult := range txResults {
 		resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult.Err)
@@ -653,7 +660,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	}
 }
 
-func aggregateTxResults(resultsByCode map[SendTxReturnCode][]error) (txResult error, err error) {
+func aggregateTxResults(resultsByCode sendTxErrors) (txResult error, err error) {
 	severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors)
 	successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes)
 	if hasSuccess {