Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps

.buildkite/scripts/cloud_tests.sh → .buildkite/scripts/setup_cloud_env.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-
 REPO_DIR=$(pwd)
 
 teardown() {
@@ -31,7 +30,6 @@ tf_cleanup() {
 trap 'teardown' EXIT
 
 # Prepare the cloud resources using Terraform
-#startCloudTestEnv "${MODULE_DIR}"
 echo "~~~ Loading creds"
 set +o xtrace
 export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
@@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci"
 export TF_VAR_REPO="${REPO}"
 terraform init && terraform apply -auto-approve
 cd -
-
-# Run tests
-echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
-cd "${BEATS_PROJECT_NAME}"
-mage build test

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -2,15 +2,15 @@
 name: "beats-xpack-filebeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
 
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -29,7 +29,7 @@ steps:
   - group: "x-pack/filebeat Mandatory Tests"
     key: "x-pack-filebeat-mandatory-tests"
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: x-pack/filebeat Unit Tests"
         key: "x-pack-filebeat-mandatory-linux-unit-test"
         command: |
           cd x-pack/filebeat
@@ -46,17 +46,13 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu Unit Tests"
+              context: "x-pack/filebeat: Unit Tests"
 
-      - label: ":go: Go (MODULE) Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Go Integration Tests"
         key: "x-pack-filebeat-mandatory-int-test"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests"
-          cd x-pack/filebeat && mage goIntegTest
+          cd x-pack/filebeat
+          mage goIntegTest
         retry:
           automatic:
             - limit: 3
@@ -69,17 +65,13 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Go (MODULE) Integration Tests"
+              context: "x-pack/filebeat: Go Integration Tests"
 
-      - label: ":python: Python (MODULE) Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Python Integration Tests"
         key: "x-pack-filebeat-mandatory-python-int-test"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests"
-          cd x-pack/filebeat && mage pythonIntegTest
+          cd x-pack/filebeat
+          mage pythonIntegTest
         retry:
           automatic:
             - limit: 3
@@ -92,9 +84,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Python (MODULE) Integration Tests"
+              context: "x-pack/filebeat: Python Integration Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -113,9 +105,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2022 Unit Tests"
+              context: "x-pack/filebeat: Win-2022 Unit Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -134,32 +126,14 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2016 Unit Tests"
-
-      - label: ":linux: Ubuntu ARM Unit Tests"
-        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
-        command: |
-          cd x-pack/filebeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu ARM Unit Tests"
+              context: "x-pack/filebeat: Win-2016 Unit Tests"
 
   - group: "x-pack/filebeat Extended Windows Tests"
     key: "x-pack-filebeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -178,9 +152,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2019 Unit Tests"
+              context: "x-pack/filebeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -199,9 +173,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 10 Unit Tests"
+              context: "x-pack/filebeat: Win 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 11 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -220,17 +194,36 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 11 Unit Tests"
+              context: "x-pack/filebeat: Win 11 Unit Tests"
 
   - group: "x-pack/filebeat Extended Tests"
     key: "x-pack-filebeat-extended-tests"
+
     steps:
-      - label: ":mac: MacOS x86_64 Unit Tests"
+      - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
+        command: |
+          cd x-pack/filebeat
+          mage build unitTest
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "x-pack/filebeat/build/*.xml"
+          - "x-pack/filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "x-pack/filebeat: Linux arm64 Unit Tests"
+
+      - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/filebeat && mage build unitTest
+          cd x-pack/filebeat
+          mage build unitTest
         retry:
           automatic:
             - limit: 3
@@ -242,18 +235,20 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: MacOS x86_64 Unit Tests"
+              context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
-        skip: "https://github.com/elastic/beats/issues/33036"
+      - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
+        skip: "Skipping due to elastic/beats#33036"
+        # https://github.com/elastic/beats/issues/33036
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/filebeat && mage build unitTest
+          cd x-pack/filebeat
+          mage build unitTest
         retry:
           automatic:
-            - limit: 3
+          - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -262,9 +257,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: MacOS arm64 Unit Tests"
+              context: "x-pack/filebeat: macOS arm64 Unit Tests"
 
-      - label: ":linux: Cloud (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test"
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
@@ -273,12 +268,12 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests"
-
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
@@ -292,26 +287,28 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Cloud (MODULE) Tests"
 
-      - label: ":linux: Cloud AWS (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test-aws"
-        skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425"
+        skip: "Skipping due to elastic/beats#36425"
+        # https://github.com/elastic/beats/issues/36425
         if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests"
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test goIntegTest
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/filebeat/build/*.xml"
@@ -331,7 +328,7 @@ steps:
   - group: "x-pack/filebeat Packaging"
     key: "x-pack-filebeat-packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":linux: x-pack/filebeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd x-pack/filebeat
@@ -352,7 +349,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":linux: x-pack/filebeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/filebeat
@@ -363,11 +360,11 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Packaging Linux ARM"
+              context: "x-pack/filebeat: Packaging Linux arm64"