updated cloud test script & xpack/filebeat xpack/metricbeat

.buildkite/scripts/cloud_tests.sh → .buildkite/scripts/setup_cloud_env.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-
 REPO_DIR=$(pwd)
 
 teardown() {
@@ -31,7 +30,6 @@ tf_cleanup() {
 trap 'teardown' EXIT
 
 # Prepare the cloud resources using Terraform
-#startCloudTestEnv "${MODULE_DIR}"
 echo "~~~ Loading creds"
 set +o xtrace
 export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
@@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci"
 export TF_VAR_REPO="${REPO}"
 terraform init && terraform apply -auto-approve
 cd -
-
-# Run tests
-echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
-cd "${BEATS_PROJECT_NAME}"
-mage build test

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -2,15 +2,15 @@
 name: "beats-xpack-filebeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
 
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -25,7 +25,7 @@ steps:
   - group: "x-pack/filebeat Mandatory Tests"
     key: "x-pack-filebeat-mandatory-tests"
     steps:
-      - label: ":ubuntu: Unit Tests"
+      - label: ":ubuntu: x-pack/filebeat Unit Tests"
         key: "x-pack-filebeat-mandatory-linux-unit-test"
         command: |
           cd x-pack/filebeat
@@ -42,9 +42,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu Unit Tests"
+              context: "x-pack/filebeat: Unit Tests"
 
-      - label: ":ubuntu: Go Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Go Integration Tests"
         key: "x-pack-filebeat-mandatory-int-test"
         command: |
           cd x-pack/filebeat
@@ -63,7 +63,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Go Integration Tests"
 
-      - label: ":ubuntu: Python Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Python Integration Tests"
         key: "x-pack-filebeat-mandatory-python-int-test"
         command: |
           cd x-pack/filebeat
@@ -82,7 +82,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Python Integration Tests"
 
-      - label: ":windows: Win-2022 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -103,7 +103,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Win-2022 Unit Tests"
 
-      - label: ":windows: Win-2016 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -129,7 +129,7 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: Win-2019 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -150,7 +150,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Win 10 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -171,7 +171,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Win 10 Unit Tests"
 
-      - label: ":windows: Win 11 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 11 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -197,24 +197,24 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|aws).*/
 
     steps:
-      - label: ":ubuntu: ARM Unit Tests"
+      - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
         command: |
           cd x-pack/filebeat
           mage build unitTest
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/filebeat/build/*.xml"
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu ARM Unit Tests"
+              context: "x-pack/filebeat: Linux arm64 Unit Tests"
 
-      - label: ":mac: macOS x86_64 Unit Tests"
+      - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
         command: |
           set -euo pipefail
@@ -234,8 +234,9 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: macOS arm64 Unit Tests"
-        skip: "https://github.com/elastic/beats/issues/33036"
+      - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
+        skip: "Skipping due to elastic/beats#33036"
+        # https://github.com/elastic/beats/issues/33036
         if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
         command: |
           set -euo pipefail
@@ -255,20 +256,22 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: macOS arm64 Unit Tests"
 
-      - label: ":ubuntu: Cloud (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
+
           echo "~~~ Running tests with env var MODULE=$$MODULE"
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
@@ -282,26 +285,29 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Cloud (MODULE) Tests"
 
-      - label: ":ubuntu: Cloud AWS (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test-aws"
-        skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425"
+        skip: "Skipping due to elastic/beats#36425"
+        # https://github.com/elastic/beats/issues/36425
         if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
+
           echo "~~~ Running tests with env var MODULE=$$MODULE"
-          .buildkite/scripts/aws_cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test goIntegTest
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/filebeat/build/*.xml"
@@ -321,7 +327,7 @@ steps:
   - group: "x-pack/filebeat Packaging"
     key: "x-pack-filebeat-packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":linux: x-pack/filebeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd x-pack/filebeat
@@ -338,18 +344,18 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":linux: x-pack/filebeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/filebeat
           mage package
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Packaging Linux ARM"
+              context: "x-pack/filebeat: Packaging Linux arm64"