build_aarch64 #4000
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2014-present, The osquery authors | |
| # | |
| # This source code is licensed as defined by the LICENSE file found in the | |
| # root directory of this source tree. | |
| # | |
| # SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) | |
| # | |
| # Due to a limitation in how GitHub Actions works, we can't reference | |
| # jobs in another file inside the `needs` statement. | |
| # | |
| # This configuration file takes care of the AArch64 Linux build | |
| name: build_aarch64 | |
| on: | |
| # Run this workflow once every 6 hours against the master branch | |
| schedule: | |
| - cron: "0 */6 * * *" | |
| # Do not run this on pull requests | |
| push: | |
| branches: | |
| - '**' | |
| tags: | |
| - '**' | |
| # Please remember to update values for both x86 and aarch64 workflows. | |
| env: | |
| PACKAGING_REPO: https://github.com/osquery/osquery-packaging | |
| PACKAGING_COMMIT: 4caa2c54f0d893c1efa47932571046bbce156c52 | |
| SUBMODULE_CACHE_VERSION: 2 | |
| # If the initial code sanity checks are passing, then one job | |
| # per [`platform` * `build_type`] will start, building osquery | |
| # and generating packages that are later attached to the commit | |
| # (or PR) as build artifacts. | |
| jobs: | |
| # This job performs basic source code check, looking for formatting | |
| # issues and missing copyright headers | |
| check_code_style: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock | |
| steps: | |
| # We are using checkout@v1 because the checkout@v2 action downloads | |
| # the source code without cloning if the installed git is < v2.18. | |
| # Once we update the image we will also be able to select the clone | |
| # destination; right now we are moving the .git folder manually. | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| # This script makes sure that the copyright headers have been correctly | |
| # placed on all the source code files | |
| - name: Check the copyright headers | |
| run: | | |
| ./tools/ci/scripts/check_copyright_headers.py | |
| - name: Setup the build paths | |
| shell: bash | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/build" | |
| rel_source_path="workspace/src" | |
| mkdir -p "${rel_build_path}" | |
| ln -sf "$(pwd)" "${rel_source_path}" | |
| echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| # Formatting is tested against the clang-format binary we ship | |
| # with the osquery-toolchain, so this job is only performed once on | |
| # a Linux machine. | |
| - name: Check code formatting | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | |
| cmake --build . --target format_check | |
| # NOTE: Here we manually create two different steps and outputs because there's no easy way | |
| # to dynamically access variables per matrix job. Having outputs with different names | |
| # lets us easily select the correct one later. | |
| # Starts self-hosted runner. | |
| start_ec2_runners: | |
| needs: check_code_style | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label-Release: ${{ steps.start_ec2_runner-Release.outputs.label }} | |
| ec2-instance-id-Release: ${{ steps.start_ec2_runner-Release.outputs.ec2-instance-id }} | |
| label-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.label }} | |
| ec2-instance-id-RelWithDebInfo: ${{ steps.start_ec2_runner-RelWithDebInfo.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} | |
| - name: Start aarch64 EC2 runner for a Release build | |
| id: start_ec2_runner-Release | |
| uses: osquery/ec2-github-runner@v2.3.3 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-063d99d9ce1bebbe8 | |
| ec2-instance-type: c6g.4xlarge | |
| subnet-id: subnet-06390365e72579736 | |
| security-group-id: sg-0757a3162c999331b | |
| - name: Start aarch64 EC2 runner for a RelWithDebInfo build | |
| id: start_ec2_runner-RelWithDebInfo | |
| uses: osquery/ec2-github-runner@v2.3.3 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-063d99d9ce1bebbe8 | |
| ec2-instance-type: c6g.4xlarge | |
| subnet-id: subnet-06390365e72579736 | |
| security-group-id: sg-0757a3162c999331b | |
| # Stops self-hosted runners. | |
| stop_ec2_runners: | |
| needs: | |
| - start_ec2_runners | |
| - build_linux | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| aws-access-key-id: ${{ secrets.EC2_GITHUB_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.EC2_GITHUB_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.EC2_GITHUB_RUNNER_AWS_REGION }} | |
| - name: Stop aarch64 EC2 runner for Release mode build | |
| uses: osquery/ec2-github-runner@v2.3.3 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start_ec2_runners.outputs.label-Release }} | |
| ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-Release }} | |
| - name: Stop aarch64 EC2 runner for RelWithDebInfo mode build | |
| uses: osquery/ec2-github-runner@v2.3.3 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.EC2_GITHUB_RUNNER_GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} | |
| ec2-instance-id: ${{ needs.start_ec2_runners.outputs.ec2-instance-id-RelWithDebInfo }} | |
| # The Linux build will only start once we know that the code | |
| # has been properly formatted | |
| build_linux: | |
| needs: start_ec2_runners | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ${{ needs.start_ec2_runners.outputs.label-Release }} | |
| build_type: Release | |
| cache_key: ubuntu-20.04_aarch64 | |
| - os: ${{ needs.start_ec2_runners.outputs.label-RelWithDebInfo }} | |
| build_type: RelWithDebInfo | |
| cache_key: ubuntu-20.04_aarch64 | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| - name: Select the build job count | |
| shell: bash | |
| id: build_job_count | |
| run: | | |
| echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT | |
| - name: Select the build options for the tests | |
| shell: bash | |
| id: tests_build_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| fi | |
| # try to gather some info about build space | |
| - name: Disk Space Information | |
| shell: bash | |
| id: disk_space_info | |
| run: | | |
| df -h | |
| # We don't have enough space on the worker to actually generate all | |
| # the debug symbols (osquery + dependencies), so we have a flag to | |
| # disable them when running a Debug build | |
| - name: Select the debug symbols options | |
| shell: bash | |
| id: debug_symbols_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| # When we spawn in the container, we are root; create an unprivileged | |
| # user now so that we can later use it to launch the normal user tests | |
| - name: Create a non-root user | |
| id: unprivileged_user | |
| run: | | |
| useradd -m -s /bin/bash unprivileged_user | |
| echo "NAME=unprivileged_user" >> $GITHUB_OUTPUT | |
| # Due to how the RPM packaging tools work, we have to adhere to some | |
| # character count requirements in the build path vs source path. | |
| # | |
| # Failing to do so, will break the debuginfo RPM package. | |
| - name: Setup the build paths | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/usr/src/debug/osquery/build" | |
| rel_src_path="workspace/padding-required-by-rpm-packages/src" | |
| rel_ccache_path="workspace/ccache" | |
| rel_package_data_path="workspace/package_data" | |
| rel_packaging_path="workspace/osquery-packaging" | |
| rel_package_build_path="workspace/package-build" | |
| mkdir -p ${rel_build_path} \ | |
| ${rel_src_path} \ | |
| ${rel_ccache_path} \ | |
| ${rel_src_path} \ | |
| ${rel_package_data_path} \ | |
| ${rel_package_build_path} | |
| chown -R ${{ steps.unprivileged_user.outputs.NAME }}:${{ steps.unprivileged_user.outputs.NAME }} . | |
| mv .git "${rel_src_path}" | |
| ( cd "${rel_src_path}" && git reset --hard ) | |
| echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT | |
| echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT | |
| - name: Clone the osquery-packaging repository | |
| run: | | |
| git clone ${{ env.PACKAGING_REPO }} \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cd ${{ steps.build_paths.outputs.PACKAGING }} | |
| git checkout ${{ env.PACKAGING_COMMIT }} | |
| # One of the tests in the test suit will spawn a Docker container | |
| # using this socket. Allow the unprivileged user we created | |
| # to access it. | |
| - name: Update the Docker socket permissions | |
| run: | | |
| chmod 666 /var/run/docker.sock | |
| - name: Update the cache (ccache) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.CCACHE }} | |
| key: | | |
| ccache_${{ matrix.cache_key }}_${{ matrix.build_type }}_${{ github.sha }} | |
| restore-keys: | | |
| ccache_${{ matrix.cache_key }}_${{ matrix.build_type }} | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules | |
| key: | | |
| gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.cache_key }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the git submodules | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| git submodule sync --recursive | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ | |
| -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| - name: Build the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} | |
| - name: Disk Space Information | |
| shell: bash | |
| id: disk_space_info_post_build | |
| run: | | |
| df -h | |
| du -sh ${{ steps.build_paths.outputs.BINARY }} | |
| - name: Run the tests as normal user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| sudo -u ${{ steps.unprivileged_user.outputs.NAME }} ctest --build-nocmake -LE "root-required" -V | |
| - name: Run the tests as root user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| sudo -u root ctest --build-nocmake -L "root-required" -V | |
| - name: Run the install target | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| run: | | |
| cmake \ | |
| --build . \ | |
| --target install \ | |
| -j ${{ steps.build_job_count.outputs.VALUE }} | |
| # Do some targeted cleanups to get additional free space before installing dependencies, | |
| # we are unfortunately out of space otherwise. | |
| find . -name "*.a" -exec rm {} \; | |
| find . -name "*.o" -exec rm {} \; | |
| # Since we need to run CMake to create the packages with osquery-packaging, the | |
| # configuration will fail unless the C and C++ compilers are found | |
| - name: Install packaging dependencies | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| run: | | |
| sudo apt update | |
| sudo apt install build-essential gcc g++ -y | |
| - name: Create the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| shell: bash | |
| run: | | |
| osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) | |
| tar pcvzf package_data.tar.gz \ | |
| ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| package_format_list=( "DEB" "RPM" "TGZ" ) | |
| for package_format in "${package_format_list[@]}" ; do | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCPACK_GENERATOR=${package_format} \ | |
| -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ | |
| -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ | |
| -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cmake --build . \ | |
| --target package | |
| done | |
| - name: Locate the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_aarch64.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Store the unsigned release package data artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_package_data_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} | |
| - name: Store the unsigned release DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_deb_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} | |
| - name: Store the unsigned debug DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_debug_deb_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} | |
| - name: Store the unsigned release RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_rpm_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} | |
| - name: Store the unsigned debug RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_debug_rpm_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} | |
| - name: Store the unsigned release TGZ artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_tgz_aarch64 | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} | |
| # Before we terminate this job, delete the build folder. The cache | |
| # actions will require the disk space to create the archives. | |
| - name: Reclaim disk space | |
| run: | | |
| rm -rf ${{ steps.build_paths.outputs.BINARY }} |