Update auditbeat pipeline to match Jenkins steps (#39245)

This commit updates the `auditbeat` pipeline to match same steps & commands as in Jenkins.
Relates: elastic/ingest-dev#3254
---------

Co-authored-by: Dimitrios Liappis <dimitrios.liappis@gmail.com>

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -3,7 +3,6 @@ name: "beats-auditbeat"
 
 env:
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
-
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
@@ -30,6 +29,7 @@ steps:
     steps:
       - label: ":ubuntu: Auditbeat Unit Tests"
         command: |
+          set -euo pipefail
           cd auditbeat
           mage build unitTest
         retry:
@@ -48,6 +48,7 @@ steps:
 
       - label: ":rhel: Auditbeat Unit Tests"
         command: |
+          set -euo pipefail
           cd auditbeat
           mage build unitTest
         retry:
@@ -118,70 +119,120 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
-              context: "auditbeat: Cross compile"
+              context: "auditbeat: Crosscompile"
+
+  - group: "Auditbeat Extended Tests"
+    key: "auditbeat-extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" ||  build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/
 
-  - group: "Auditbeat Linux arm64 Tests"
-    key: "auditbeat-extended-tests-linux-arm64"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
     steps:
-      - label: ":linux: Auditbeat Linux arm64 Unit Tests"
+      - label: ":linux: Auditbeat Ubuntu Integration Tests"
+        key: "auditbeat-extended-integ-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
+        command: |
+          set -euo pipefail
+          cd auditbeat
+          mage build integTest
+        retry:
+          automatic:
+           - limit: 3
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Linux Integration Tests"
+
+      - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests"
+        key: "auditbeat-extended-arm64-integ-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
+        command: |
+          set -euo pipefail
+          cd auditbeat
+          mage build integTest
+        retry:
+          automatic:
+           - limit: 3
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Linux arm64 Integration Tests"
+
+      - label: ":linux: Auditbeat Ubuntu arm64 Unit Tests"
         key: "auditbeat-extended-arm64-unit-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
           set -euo pipefail
           cd auditbeat
-          mage unitTest
+          mage build unitTest
         retry:
           automatic:
            - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Linux arm64 tests"
+              context: "auditbeat: Linux arm64 Unit Tests"
 
-  - group: "Auditbeat MacOS Extended"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
       - label: ":mac: Auditbeat macOS x86_64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
-          mage unitTest
+          mage build unitTest
         retry:
           automatic:
            - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "auditbeat: Extended MacOS x86_64 Unit Tests"
+              context: "auditbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: Auditbeat macOS arm64 Unit Tests"
+      - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
-          mage unitTest
+          mage build unitTest
         retry:
           automatic:
            - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "auditbeat: Extended MacOS arm64 Unit Tests"
+              context: "auditbeat: macOS arm64 Unit Tests"
 
   - group: "Auditbeat Windows Extended Testing"
     key: "auditbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
       - label: ":windows: Auditbeat Win-2019 Unit Tests"
         key: "auditbeat-extended-win-2019-unit-tests"
@@ -197,7 +248,9 @@ steps:
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 2019 Unit Tests"
@@ -216,7 +269,9 @@ steps:
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 10 Unit Tests"
@@ -235,7 +290,9 @@ steps:
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "auditbeat/build/*.*"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Extended Windows 11 Unit Tests"
@@ -256,6 +313,7 @@ steps:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd auditbeat
           mage package
         agents:
@@ -273,6 +331,7 @@ steps:
           PACKAGES: "docker"
           SNAPSHOT: true
         command: |
+          set -euo pipefail
           cd auditbeat
           mage package
         agents:

.buildkite/hooks/pre-command

@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   source .buildkite/env-scripts/env.sh
   if [[ -z "${GO_VERSION-""}" ]]; then
     export GO_VERSION=$(cat "${WORKSPACE}/.go-version")