[7.17](backport #39126) Beats DRA pipeline

.buildkite/packaging.pipeline.md

@@ -0,0 +1,23 @@
+### Beats Packaging pipeline
+[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches.
+The pipeline definition is located in the `.buildkite/packaging.pipeline.yml`
+
+### Triggers
+Staging packaging DRA is triggered for release branches only.
+Snapshot is triggered for `main` and release branches.
+
+### Pipeline steps
+
+#### Beats dashboard
+Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration.
+
+#### Packaging snapshot/staging
+
+- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`)
+- Copies artifacts `build/distributions/<beat>/` directory and adds it as an artifact, where `<beat>` is the corresponding beat name.
+- x-pack artifacts are also copied to `build/distributions/<beat>/` directory, where `<beat>` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle).
+
+#### DRA publish
+Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry.
+
+

.buildkite/packaging.pipeline.yml

@@ -0,0 +1,248 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+# TODO: Pre-cache beats-dev/golang-crossbuild container image
+
+env:
+  ASDF_MAGE_VERSION: 1.15.0
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+
+  PLATFORMS: "+all linux/amd64 linux/386 linux/arm64 windows/amd64 windows/386 darwin/amd64"
+  PLATFORMS_ARM: "linux/arm64"
+
+steps:
+  # we use concurrency gates (https://buildkite.com/blog/concurrency-gates)
+  # to implement two FIFO queues for DRA-snapshot and DRA-staging
+  # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second
+
+  - name: Start of concurrency group for DRA Snapshot
+    if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+    command: echo "--> Start of concurrency gate dra-snapshot"
+    concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
+    concurrency: 1
+    key: start-gate-snapshot
+
+  - name: Start of concurrency group for DRA Staging
+    if: build.branch =~ /^\d+\.\d+$$/
+    command: echo "--> Start of concurrency gate dra-staging"
+    concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
+    concurrency: 1
+    key: start-gate-staging
+
+  - wait
+
+  - group: Beats dashboards
+    key: dashboards
+    steps:
+      - label: Snapshot dashboards
+        if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+        depends_on: start-gate-snapshot
+        key: dashboards-snapshot
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        env:
+          SNAPSHOT: true
+          DEV: true
+        artifact_paths:
+          - build/distributions/**/*
+
+      - label: Staging dashboards
+        if: build.branch =~ /^\d+\.\d+$$/
+        depends_on: start-gate-staging
+        key: dashboards-staging
+        # TODO: container with go and make
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        commands:
+          - make build/distributions/dependencies.csv
+          - make beats-dashboards
+        env:
+          SNAPSHOT: false
+          DEV: false
+        artifact_paths:
+          - build/distributions/**/*
+
+  - group: Packaging snapshot
+    if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+    key: packaging-snapshot
+    depends_on: start-gate-snapshot
+    steps:
+      - label: "SNAPSHOT: {{matrix}}"
+        env:
+          PLATFORMS: "${PLATFORMS}"
+          SNAPSHOT: true
+          DEV: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/osquerybeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      - label: "SNAPSHOT: {{matrix}} docker Linux/arm64"
+        env:
+          PLATFORMS: "${PLATFORMS_ARM}"
+          PACKAGES: "docker"
+          SNAPSHOT: true
+          DEV: true
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+  - group: Packaging staging
+    key: packaging-staging
+    depends_on: start-gate-staging
+    ## Only for release
+    if: build.branch =~ /^\d+\.\d+$$/
+    steps:
+      - label: "STAGING: {{matrix}}"
+        env:
+          PLATFORMS: "${PLATFORMS}"
+          SNAPSHOT: false
+          DEV: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/osquerybeat
+          - x-pack/packetbeat
+          - x-pack/winlogbeat
+
+      - label: "STAGING: {{matrix}} docker Linux/arm64"
+        env:
+          PLATFORMS: "${PLATFORMS_ARM}"
+          PACKAGES: "docker"
+          SNAPSHOT: false
+          DEV: false
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
+
+  - group: DRA publish
+    key: dra
+    steps:
+      - label: DRA Snapshot
+        ## Only for release branches and main
+        if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+        key: dra-snapshot
+        env:
+          DRA_WORKFLOW: snapshot
+        depends_on:
+          - start-gate-snapshot
+          - packaging-snapshot
+          - dashboards-snapshot
+        command: |
+          buildkite-agent artifact download "build/**/*" . 
+          .buildkite/scripts/packaging/prepare-release-manager.sh snapshot
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+
+      - label: DRA Staging
+        ## Only for release branches
+        if: build.branch =~ /^\d+\.\d+$$/
+        key: dra-staging
+        env:
+          DRA_WORKFLOW: staging
+        depends_on:
+          - start-gate-staging
+          - packaging-staging
+          - dashboards-staging
+        command: |
+          buildkite-agent artifact download "build/**" .
+          .buildkite/scripts/packaging/prepare-release-manager.sh staging
+          .buildkite/scripts/dra.sh
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+
+  - wait
+
+  - command: echo "End of concurrency gate dra-snapshot <--"
+    if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+    concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
+    concurrency: 1
+    key: end-gate-snapshot
+
+  - command: echo "End of concurrency gate dra-staging <--"
+    if: build.branch =~ /^\d+\.\d+$$/
+    concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
+    concurrency: 1
+    key: end-gate-staging

.buildkite/scripts/dra.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# TODO: uncomment out below when Jenkins packaging has been stopped
+# if [[ "$DRY_RUN" == "true" ]]; then
+#     echo "~~~ Running in dry-run mode -- will NOT publish artifacts"
+#     DRY_RUN="--dry-run"
+# else
+#     echo "~~~ Running in publish mode"
+#     DRY_RUN=""
+# fi
+
+# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped
+if [[ "$DRY_RUN" == "false" ]]; then
+    echo "~~~ Running in publish mode"
+    DRY_RUN=""
+else
+    echo "~~~ Running in dry-run mode -- will NOT publish artifacts"
+    DRY_RUN="--dry-run"
+fi
+
+set -euo pipefail
+
+# DRA_BRANCH can be used for manually testing packaging with PRs
+# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
+
+BEAT_VERSION=$(make get-version)
+
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+set +x
+release_manager_login
+
+# required by the release-manager docker image, otherwise we hit:
+# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied)
+chmod -R a+r build/*
+chmod -R a+w build
+
+echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]"
+set +x
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli list \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main"
+
+echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]"
+
+set +x
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project "beats" \
+        --branch "${BRANCH}" \
+        --commit "${BUILDKITE_COMMIT}" \
+        --workflow "${DRA_WORKFLOW}" \
+        --version "${BEAT_VERSION}" \
+        --artifact-set "main" \
+        ${DRY_RUN} | tee rm-output.txt
+
+
+if [[ "$DRY_RUN" != "--dry-run" ]]; then
+  # extract the summary URL from a release manager output line like:
+  # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html
+  SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}')
+  rm rm-output.txt
+
+  # and make it easily clickable as a Builkite annotation
+  printf "**${DRA_WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
+fi

.buildkite/scripts/packaging/package-dra.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+BEAT_DIR=${1:-""}
+
+if [ -z "$BEAT_DIR" ]; then
+    echo "Error: Beat directory must be specified."
+    exit 1
+fi
+
+echo "~~~ Packaging : $BEAT_DIR"
+
+WORKSPACE=$(pwd)
+BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')
+
+cd $BEAT_DIR
+mage package
+mage ironbank
+
+mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG
+cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/
+cd $WORKSPACE