Skip to content

Commit

Permalink
updated xpack-filebeat pipeleine to match Jenkinsfile
Browse files Browse the repository at this point in the history
  • Loading branch information
oakrizan committed Apr 26, 2024
1 parent 9649588 commit 80030ff
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 47 deletions.
57 changes: 57 additions & 0 deletions .buildkite/scripts/aws_cloud_tests.sh
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
set -euo pipefail


REPO_DIR=$(pwd)

teardown() {
# reset the directory to the root of the project
cd $REPO_DIR
# Teardown resources after using them
echo "~~~ Terraform Cleanup"
tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK

echo "~~~ Docker Compose Cleanup"
docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
}

tf_cleanup() {
DIRECTORY=${1:-.}

for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
cd $(dirname $tfstate)
terraform init
if ! terraform destroy -auto-approve; then
echo "+++ Failed to Terraform destroy the resources"
fi
cd -
done
}

trap 'teardown' EXIT

# Prepare the cloud resources using Terraform
#startCloudTestEnv "${MODULE_DIR}"
echo "~~~ Loading creds"
set +o xtrace
export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY
export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
set -o xtrace

echo "~~~ Run docker-compose services for emulated cloud env"
docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
echo "~~~ Initialize TF cloud resources"
cd "$MODULE_DIR"
export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
export TF_VAR_CREATED_DATE=$(date +%s)
export TF_VAR_ENVIRONMENT="ci"
export TF_VAR_REPO="${REPO}"
terraform init && terraform apply -auto-approve
cd -

# Run tests
echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
cd "${BEATS_PROJECT_NAME}"
mage build test goIntegTest
94 changes: 47 additions & 47 deletions .buildkite/x-pack/pipeline.xpack.filebeat.yml
Expand Up @@ -25,9 +25,10 @@ steps:
- group: "x-pack/filebeat Mandatory Tests"
key: "x-pack-filebeat-mandatory-tests"
steps:
- label: ":linux: Ubuntu Unit Tests"
- label: ":ubuntu: Unit Tests"
key: "x-pack-filebeat-mandatory-linux-unit-test"
command: |
set -euo pipefail
cd x-pack/filebeat
mage build unitTest
agents:
Expand All @@ -41,15 +42,12 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: Ubuntu Unit Tests"

- label: ":go: Go (MODULE) Integration Tests"
- label: ":ubuntu: Go Integration Tests"
key: "x-pack-filebeat-mandatory-int-test"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Will run tests with env var MODULE=$$MODULE"
cd x-pack/filebeat && mage goIntegTest
cd x-pack/filebeat
mage goIntegTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
Expand All @@ -59,17 +57,14 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Go (MODULE) Integration Tests"
context: "x-pack/filebeat: Go Integration Tests"

- label: ":python: Python (MODULE) Integration Tests"
- label: ":ubuntu: Python Integration Tests"
key: "x-pack-filebeat-mandatory-python-int-test"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests with env var MODULE=$$MODULE"
cd x-pack/filebeat && mage pythonIntegTest
cd x-pack/filebeat
mage pythonIntegTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
Expand All @@ -79,9 +74,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Python (MODULE) Integration Tests"
context: "x-pack/filebeat: Python Integration Tests"

- label: ":windows: Windows 2022 Unit Tests"
- label: ":windows: Win-2022 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -97,9 +92,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2022 Unit Tests"
context: "x-pack/filebeat: Win-2022 Unit Tests"

- label: ":windows: Windows 2016 Unit Tests"
- label: ":windows: Win-2016 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -115,29 +110,13 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2016 Unit Tests"

- label: ":linux: Ubuntu ARM Unit Tests"
key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
command: |
cd x-pack/filebeat
mage build unitTest
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "x-pack/filebeat/build/*.xml"
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Ubuntu ARM Unit Tests"
context: "x-pack/filebeat: Win-2016 Unit Tests"

- group: "x-pack/filebeat Extended Windows Tests"
key: "x-pack-filebeat-extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
steps:
- label: ":windows: Windows 2019 Unit Tests"
- label: ":windows: Win-2019 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -153,9 +132,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 2019 Unit Tests"
context: "x-pack/filebeat: Win-2019 Unit Tests"

- label: ":windows: Windows 10 Unit Tests"
- label: ":windows: Win 10 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -171,9 +150,9 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 10 Unit Tests"
context: "x-pack/filebeat: Win 10 Unit Tests"

- label: ":windows: Windows 11 Unit Tests"
- label: ":windows: Win 11 Unit Tests"
command: |
Set-Location -Path x-pack/filebeat
mage build unitTest
Expand All @@ -189,17 +168,37 @@ steps:
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Windows 11 Unit Tests"
context: "x-pack/filebeat: Win 11 Unit Tests"

- group: "x-pack/filebeat Extended Tests"
key: "x-pack-filebeat-extended-tests"
steps:

- label: ":ubuntu: ARM Unit Tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
command: |
set -euo pipefail
cd x-pack/filebeat
mage build unitTest
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "x-pack/filebeat/build/*.xml"
- "x-pack/filebeat/build/*.json"
notify:
- github_commit_status:
context: "x-pack/filebeat: Ubuntu ARM Unit Tests"

- label: ":mac: MacOS x86_64 Unit Tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd x-pack/filebeat && mage build unitTest
cd x-pack/filebeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_X86_64}"
Expand All @@ -210,13 +209,14 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: MacOS x86_64 Unit Tests"

- label: ":mac: MacOS arm64 Unit Tests"
- label: ":mac: MacOS ARM Unit Tests"
skip: "https://github.com/elastic/beats/issues/33036"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd x-pack/filebeat && mage build unitTest
cd x-pack/filebeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_ARM}"
Expand All @@ -227,7 +227,7 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: MacOS arm64 Unit Tests"

- label: ":linux: Cloud (MODULE) Tests"
- label: ":ubuntu: Cloud (MODULE) Tests"
key: "x-pack-filebeat-extended-cloud-test"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
command: |
Expand Down Expand Up @@ -255,7 +255,7 @@ steps:
- github_commit_status:
context: "x-pack/filebeat: Cloud (MODULE) Tests"

- label: ":linux: Cloud AWS (MODULE) Tests"
- label: ":ubuntu: Cloud AWS (MODULE) Tests"
key: "x-pack-filebeat-extended-cloud-test-aws"
skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
Expand All @@ -265,7 +265,7 @@ steps:
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet x-pack/filebeat
echo "~~~ Running tests with env var MODULE=$$MODULE"
.buildkite/scripts/cloud_tests.sh
.buildkite/scripts/aws_cloud_tests.sh
env:
ASDF_TERRAFORM_VERSION: 1.0.2
AWS_REGION: "eu-central-1"
Expand Down

0 comments on commit 80030ff

Please sign in to comment.