build_x86 #5459
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2014-present, The osquery authors | |
| # | |
| # This source code is licensed as defined by the LICENSE file found in the | |
| # root directory of this source tree. | |
| # | |
| # SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) | |
| # | |
| # Due to a limitation in how GitHub Actions works, we can't reference | |
| # jobs in another file inside the `needs` statement. | |
| # | |
| # This configuration file takes care of the Windows, macOS and Linux | |
| # builds on the x86 platform. | |
| name: build_x86 | |
| on: | |
| # Run this workflow once every 6 hours against the master branch | |
| schedule: | |
| - cron: "0 */6 * * *" | |
| push: | |
| branches: | |
| - 'master' | |
| - 'main' | |
| tags: | |
| - '*' | |
| pull_request: | |
| branches: | |
| - '*' | |
| # Cancel old jobs of a PR if a new job is started. | |
| # Fallback on using the run id if it's not a PR, which is unique, so no job canceling. | |
| concurrency: | |
| group: hosted-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| # Please remember to update values for both x86 and aarch64 workflows. | |
| env: | |
| PACKAGING_REPO: https://github.com/osquery/osquery-packaging | |
| PACKAGING_COMMIT: 4caa2c54f0d893c1efa47932571046bbce156c52 | |
| SUBMODULE_CACHE_VERSION: 2 | |
| # If the initial code sanity checks are passing, then one job | |
| # per [`platform` * `build_type`] will start, building osquery | |
| # and generating packages that are later attached to the commit | |
| # (or PR) as build artifacts. | |
| jobs: | |
| # This job performs basic source code check, looking for formatting | |
| # issues and missing copyright headers | |
| check_code_style: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock | |
| steps: | |
| # We are using checkout@v1 because the checkout@v2 action downloads | |
| # the source code without cloning if the installed git is < v2.18. | |
| # Once we update the image we will also be able to select the clone | |
| # destination; right now we are moving the .git folder manually. | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| # This script makes sure that the copyright headers have been correctly | |
| # placed on all the source code files | |
| - name: Check the copyright headers | |
| run: | | |
| ./tools/ci/scripts/check_copyright_headers.py | |
| - name: Setup the build paths | |
| shell: bash | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/build" | |
| rel_source_path="workspace/src" | |
| mkdir -p "${rel_build_path}" | |
| ln -sf "$(pwd)" "${rel_source_path}" | |
| echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DOSQUERY_ENABLE_FORMAT_ONLY=ON \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| # Formatting is tested against the clang-format binary we ship | |
| # with the osquery-toolchain, so this job is only performed once on | |
| # a Linux machine. | |
| - name: Check code formatting | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | |
| cmake --build . --target format_check | |
| # This jobs checks that the third party libraries manifest has the correct format | |
| # and that is up to date compared to the current state of the repository | |
| check_libraries_manifest: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v3 | |
| - name: Install python pre-requisites | |
| run: | | |
| pip3 install -r ./tools/ci/scripts/cve/requirements.txt | |
| - name: Verify the third party libraries manifest | |
| run: | | |
| ./tools/ci/scripts/cve/validate_manifest_libraries_versions.py --manifest libraries/third_party_libraries_manifest.json \ | |
| --repository . | |
| # Test generating the website json | |
| check_genwebsitejson: | |
| needs: [check_code_style, check_libraries_manifest] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| - name: genwebsitejson.py | |
| run: python3 tools/codegen/genwebsitejson.py --specs=specs/ | |
| # This job runs source code analysis tools (currently, just cppcheck) | |
| check_source_code: | |
| needs: [check_code_style, check_libraries_manifest] | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock | |
| strategy: | |
| matrix: | |
| os: [ubuntu-20.04] | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| - name: Setup the build paths | |
| shell: bash | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/build" | |
| rel_source_path="workspace/src" | |
| rel_install_path="workspace/install" | |
| mkdir -p ${rel_build_path} \ | |
| ${rel_source_path} \ | |
| ${rel_install_path} | |
| mv .git "${rel_source_path}" | |
| ( cd "${rel_source_path}" && git reset --hard ) | |
| echo "SOURCE=$(realpath ${rel_source_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| echo "REL_BINARY=${rel_build_path}" >> $GITHUB_OUTPUT | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules | |
| key: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the git submodules | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| git submodule sync --recursive | |
| - name: Configure the project (Release) | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DCMAKE_BUILD_TYPE:STRING=Release \ | |
| -DOSQUERY_BUILD_TESTS=ON \ | |
| -DOSQUERY_BUILD_ROOT_TESTS=ON \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| - name: Initialize the project (Release) | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake --build . --target prepare_for_ide | |
| - name: Run cppcheck (Release) | |
| shell: bash | |
| id: release_cppcheck_runner | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake --build . --target cppcheck 2>&1 | tee cppcheck_release.txt | |
| - name: Store the cppcheck log (Release) | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: cppcheck-release | |
| path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_release.txt | |
| - name: Configure the project (Debug) | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DCMAKE_BUILD_TYPE:STRING=Debug \ | |
| -DOSQUERY_BUILD_TESTS=ON \ | |
| -DOSQUERY_BUILD_ROOT_TESTS=ON \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| - name: Initialize the project (Debug) | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake --build . --target prepare_for_ide | |
| - name: Run cppcheck (Debug) | |
| shell: bash | |
| id: debug_cppcheck_runner | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| cmake --build . --target cppcheck 2>&1 | tee cppcheck_debug.txt | |
| - name: Store the cppcheck log (Debug) | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: cppcheck-debug | |
| path: ${{ steps.build_paths.outputs.REL_BINARY }}/cppcheck_debug.txt | |
| # The Linux build will only start once we know that the code | |
| # has been properly formatted | |
| build_linux: | |
| needs: [check_code_style, check_libraries_manifest] | |
| runs-on: ${{ matrix.os }} | |
| container: | |
| image: osquery/builder18.04:c7a9d706d | |
| options: --privileged --init -v /var/run/docker.sock:/var/run/docker.sock --pid=host | |
| strategy: | |
| matrix: | |
| build_type: [Release, RelWithDebInfo, Debug] | |
| os: [ubuntu-20.04] | |
| steps: | |
| - name: Make space uninstalling packages | |
| shell: bash | |
| run: | | |
| run_on_host="nsenter -t 1 -m -u -n -i" | |
| packages_to_remove=$($run_on_host dpkg-query -f '${Package}\n' -W | grep "^clang-.*\|^llvm-.*\|^php.*\|^mono-.*\|^mongodb-.*\ | |
| \|^libmono-.*\|^temurin-8-jdk\|^temurin-11-jdk\|^temurin-17-jdk\|^dotnet-.*\|^google-chrome-stable\|^microsoft-edge-stable\|^google-cloud-sdk\|^firefox\|^hhvm\|^snapd") | |
| $run_on_host apt purge $packages_to_remove | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v1 | |
| - name: Select the build job count | |
| shell: bash | |
| id: build_job_count | |
| run: | | |
| echo "VALUE=$(($(nproc) + 1))" >> $GITHUB_OUTPUT | |
| - name: Select the build options for the tests | |
| shell: bash | |
| id: tests_build_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "RelWithDebInfo" ]] ; then | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Get runner hardware info | |
| shell: bash | |
| id: runner_stats | |
| run: | | |
| df -h | |
| echo "Cores: $(nproc)" | |
| free -m | |
| # We don't have enough space on the worker to actually generate all | |
| # the debug symbols (osquery + dependencies), so we have a flag to | |
| # disable them when running a Debug build | |
| - name: Select the debug symbols options | |
| shell: bash | |
| id: debug_symbols_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| # When we spawn in the container, we are root; create an unprivileged | |
| # user now so that we can later use it to launch the normal user tests | |
| - name: Create a non-root user | |
| if: matrix.build_type != 'RelWithDebInfo' | |
| id: unprivileged_user | |
| run: | | |
| useradd -m -s /bin/bash unprivileged_user | |
| echo "NAME=unprivileged_user" >> $GITHUB_OUTPUT | |
| # Due to how the RPM packaging tools work, we have to adhere to some | |
| # character count requirements in the build path vs source path. | |
| # | |
| # Failing to do so, will break the debuginfo RPM package. | |
| - name: Setup the build paths | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/usr/src/debug/osquery/build" | |
| rel_src_path="workspace/padding-required-by-rpm-packages/src" | |
| rel_ccache_path="workspace/ccache" | |
| rel_package_data_path="workspace/package_data" | |
| rel_packaging_path="workspace/osquery-packaging" | |
| rel_package_build_path="workspace/package-build" | |
| mkdir -p ${rel_build_path} \ | |
| ${rel_src_path} \ | |
| ${rel_ccache_path} \ | |
| ${rel_src_path} \ | |
| ${rel_package_data_path} \ | |
| ${rel_package_build_path} | |
| chown -R ${{ steps.unprivileged_user.outputs.NAME }}:${{ steps.unprivileged_user.outputs.NAME }} . | |
| mv .git "${rel_src_path}" | |
| ( cd "${rel_src_path}" && git reset --hard ) | |
| echo "SOURCE=$(realpath ${rel_src_path})" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(realpath ${rel_build_path})" >> $GITHUB_OUTPUT | |
| echo "CCACHE=$(realpath ${rel_ccache_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGING=$(realpath ${rel_packaging_path})" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_DATA=$(realpath ${rel_package_data_path})" >> $GITHUB_OUTPUT | |
| echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_BUILD=$(realpath ${rel_package_build_path})" >> $GITHUB_OUTPUT | |
| - name: Clone the osquery-packaging repository | |
| run: | | |
| git clone ${{ env.PACKAGING_REPO }} \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cd ${{ steps.build_paths.outputs.PACKAGING }} | |
| git checkout ${{ env.PACKAGING_COMMIT }} | |
| # One of the tests in the test suit will spawn a Docker container | |
| # using this socket. Allow the unprivileged user we created | |
| # to access it. | |
| - name: Update the Docker socket permissions | |
| if: matrix.build_type != 'RelWithDebInfo' | |
| run: | | |
| chmod 666 /var/run/docker.sock | |
| - name: Update the cache (ccache) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.CCACHE }} | |
| key: | | |
| ccache_${{ matrix.os }}_${{ matrix.build_type }}_${{ github.sha }} | |
| restore-keys: | | |
| ccache_${{ matrix.os }}_${{ matrix.build_type }} | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules | |
| key: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the git submodules | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| git submodule sync --recursive | |
| - name: Configure the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ | |
| -DOSQUERY_TOOLCHAIN_SYSROOT:PATH="/usr/local/osquery-toolchain" \ | |
| -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ | |
| -DOSQUERY_BUILD_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| -DOSQUERY_BUILD_ROOT_TESTS=${{ steps.tests_build_settings.outputs.VALUE }} \ | |
| "${{ steps.build_paths.outputs.SOURCE }}" | |
| - name: Build the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} | |
| - name: Disk space information | |
| shell: bash | |
| id: disk_space_info_post_build | |
| run: | | |
| df -h | |
| du -sh ${{ steps.build_paths.outputs.BINARY }} | |
| # Only run the tests on Debug and Release configurations; skip RelWithDebInfo | |
| - name: Run the tests as normal user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| if: matrix.build_type != 'RelWithDebInfo' | |
| run: | | |
| sudo -u ${{ steps.unprivileged_user.outputs.NAME }} ctest --build-nocmake -LE "root-required" -V | |
| - name: Run the tests as root user | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| if: matrix.build_type != 'RelWithDebInfo' | |
| run: | | |
| sudo -u root ctest --build-nocmake -L "root-required" -V | |
| - name: Run the install target | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| run: | | |
| cmake \ | |
| --build . \ | |
| --target install \ | |
| -j ${{ steps.build_job_count.outputs.VALUE }} | |
| # Since we need to run CMake to create the packages with osquery-packaging, the | |
| # configuration will fail unless the C and C++ compilers are found | |
| - name: Install packaging dependencies | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| run: | | |
| sudo apt update | |
| sudo apt install build-essential gcc g++ -y | |
| - name: Create the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| shell: bash | |
| run: | | |
| osquery_version=$(cd ${{ steps.build_paths.outputs.SOURCE }} && git describe --tags --always ) | |
| tar pcvzf package_data.tar.gz \ | |
| ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| package_format_list=( "DEB" "RPM" "TGZ" ) | |
| for package_format in "${package_format_list[@]}" ; do | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCPACK_GENERATOR=${package_format} \ | |
| -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ | |
| -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} \ | |
| -DOSQUERY_SOURCE_DIRECTORY_LIST="${{ steps.build_paths.outputs.SOURCE }};${{ steps.build_paths.outputs.BINARY }}" \ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| cmake --build . \ | |
| --target package | |
| done | |
| - name: Locate the packages | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.deb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_DEB_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*.ddeb)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-?.*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_DEBUG_RPM_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/osquery-debuginfo-*.rpm)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/*linux_x86_64.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Store the unsigned release package data artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_package_data | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} | |
| - name: Store the unsigned release DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_deb | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_DEB_PATH }} | |
| - name: Store the unsigned debug DEB artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_debug_deb | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_DEB_PATH }} | |
| - name: Store the unsigned release RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_rpm | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_RPM_PATH }} | |
| - name: Store the unsigned debug RPM artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_debug_rpm | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_DEBUG_RPM_PATH }} | |
| - name: Store the unsigned release TGZ artifact | |
| if: matrix.build_type == 'RelWithDebInfo' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: linux_unsigned_release_tgz | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} | |
| # Before we terminate this job, delete the build folder. The cache | |
| # actions will require the disk space to create the archives. | |
| - name: Reclaim disk space | |
| run: | | |
| rm -rf ${{ steps.build_paths.outputs.BINARY }} | |
| # The macOS build will only start once we know that the code | |
| # has been properly formatted | |
| build_macos: | |
| needs: [check_code_style, check_libraries_manifest] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| build_type: [Release, Debug] | |
| architecture: [x86_64, arm64] | |
| os: [macos-12] | |
| exclude: | |
| - build_type: Debug | |
| architecture: arm64 | |
| steps: | |
| - name: Select the build job count | |
| shell: bash | |
| id: build_job_count | |
| run: | | |
| echo "VALUE=$(($(sysctl -n hw.logicalcpu) + 1))" >> $GITHUB_OUTPUT | |
| - name: Setup the build paths | |
| shell: bash | |
| id: build_paths | |
| run: | | |
| rel_build_path="workspace/build" | |
| rel_src_path="workspace/src" | |
| rel_ccache_path="workspace/ccache" | |
| rel_downloads_path="workspace/downloads" | |
| rel_install_path="workspace/install" | |
| rel_package_data_path="workspace/package_data" | |
| rel_packaging_path="workspace/osquery-packaging" | |
| rel_package_build_path="workspace/package-build" | |
| mkdir -p ${rel_build_path} \ | |
| ${rel_ccache_path} \ | |
| ${rel_downloads_path} \ | |
| ${rel_install_path} \ | |
| ${rel_package_data_path} \ | |
| ${rel_package_build_path} | |
| echo "SOURCE=$(pwd)/${rel_src_path}" >> $GITHUB_OUTPUT | |
| echo "REL_SOURCE=${rel_src_path}" >> $GITHUB_OUTPUT | |
| echo "BINARY=$(pwd)/${rel_build_path}" >> $GITHUB_OUTPUT | |
| echo "CCACHE=$(pwd)/${rel_ccache_path}" >> $GITHUB_OUTPUT | |
| echo "DOWNLOADS=$(pwd)/${rel_downloads_path}" >> $GITHUB_OUTPUT | |
| echo "INSTALL=$(pwd)/${rel_install_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGING=$(pwd)/${rel_packaging_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_DATA=$(pwd)/${rel_package_data_path}" >> $GITHUB_OUTPUT | |
| echo "REL_PACKAGE_BUILD=${rel_package_build_path}" >> $GITHUB_OUTPUT | |
| echo "PACKAGE_BUILD=$(pwd)/${rel_package_build_path}" >> $GITHUB_OUTPUT | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| path: ${{ steps.build_paths.outputs.REL_SOURCE }} | |
| - name: Get runner hardware info | |
| shell: bash | |
| id: runner_stats | |
| run: | | |
| df -h | |
| echo "Cores: $(sysctl -n hw.logicalcpu)" | |
| vm_stat | |
| - name: Update the cache (ccache) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.CCACHE }} | |
| key: | | |
| ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }}_${{ github.sha }} | |
| restore-keys: | | |
| ccache_${{ matrix.os }}_${{ matrix.architecture }}_${{ matrix.build_type }} | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}/.git/modules | |
| key: | | |
| gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.os }}_${{ matrix.architecture }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the cache (downloads) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.DOWNLOADS }} | |
| key: | | |
| downloads_${{ matrix.os }}_${{ matrix.architecture }}_${{ github.sha }} | |
| restore-keys: | | |
| downloads_${{ matrix.os }}_${{ matrix.architecture }} | |
| - name: Update the git submodules | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| run: | | |
| git submodule sync --recursive | |
| - name: Install build dependencies | |
| run: | | |
| brew install \ | |
| ccache \ | |
| flex \ | |
| bison \ | |
| coreutils \ | |
| gnu-sed | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.2' | |
| architecture: 'x64' | |
| - name: Install tests dependencies | |
| id: install_test_deps | |
| run: | | |
| pip3 install setuptools \ | |
| pexpect==3.3 \ | |
| psutil \ | |
| timeout_decorator \ | |
| six \ | |
| thrift==0.11.0 \ | |
| osquery | |
| - name: Install CMake | |
| shell: bash | |
| run: | | |
| ${{ steps.build_paths.outputs.REL_SOURCE }}/tools/ci/scripts/macos/install_cmake.sh \ | |
| "${{ steps.build_paths.outputs.DOWNLOADS }}" \ | |
| "${{ steps.build_paths.outputs.INSTALL }}" \ | |
| "3.21.4" | |
| - name: Select the Xcode version | |
| shell: bash | |
| id: xcode_selector | |
| run: | | |
| xcode_path="/Applications/Xcode_14.1.app/Contents/Developer" | |
| echo "PATH=${path}" >> $GITHUB_OUTPUT | |
| sudo xcode-select -s "${xcode_path}" | |
| echo "DEPLOYMENT_TARGET=10.15" >> $GITHUB_OUTPUT | |
| # We don't have enough space on the worker to actually generate all | |
| # the debug symbols (osquery + dependencies), so we have a flag to | |
| # disable them when running a Debug build | |
| - name: Select the debug symbols options | |
| shell: bash | |
| id: debug_symbols_settings | |
| run: | | |
| if [[ "${{ matrix.build_type }}" == "Debug" ]] ; then | |
| echo "VALUE=ON" >> $GITHUB_OUTPUT | |
| else | |
| echo "VALUE=OFF" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Configure the project | |
| shell: bash | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake -G "Unix Makefiles" \ | |
| -DCMAKE_C_COMPILER=clang \ | |
| -DCMAKE_CXX_COMPILER=clang++ \ | |
| -DCMAKE_OSX_ARCHITECTURES="${{ matrix.architecture }}" \ | |
| -DCMAKE_OSX_DEPLOYMENT_TARGET="${{ steps.xcode_selector.outputs.DEPLOYMENT_TARGET }}" \ | |
| -DCMAKE_BUILD_TYPE:STRING="${{ matrix.build_type }}" \ | |
| -DOSQUERY_BUILD_TESTS=ON \ | |
| -DOSQUERY_NO_DEBUG_SYMBOLS=${{ steps.debug_symbols_settings.outputs.VALUE }} \ | |
| ${{ steps.build_paths.outputs.SOURCE }} | |
| - name: Build the project | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| run: | | |
| cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} | |
| - name: Disk space information | |
| shell: bash | |
| id: disk_space_info_post_build | |
| run: | | |
| df -h | |
| du -sh ${{ steps.build_paths.outputs.BINARY }} | |
| - name: Run the tests | |
| if: matrix.architecture == 'x86_64' | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| run: | | |
| ctest --build-nocmake -V | |
| - name: Run the install target | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.CCACHE }} | |
| DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| run: | | |
| cmake \ | |
| --build . \ | |
| --target install \ | |
| -j ${{ steps.build_job_count.outputs.VALUE }} | |
| - name: Create the package data | |
| working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| run: | | |
| tar pcvzf package_data.tar.gz \ | |
| -C ${{ steps.build_paths.outputs.PACKAGE_DATA }} \ | |
| . | |
| - name: Locate the package data | |
| if: matrix.build_type == 'Release' | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$(ls ${{ steps.build_paths.outputs.REL_PACKAGE_BUILD }}/package_data.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Store the ${{ matrix.architecture }} unsigned release package data artifact | |
| if: matrix.build_type == 'Release' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_unsigned_release_package_data_${{ matrix.architecture }} | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} | |
| - name: Package the tests for the x86_64 macOS-11 worker | |
| if: matrix.architecture == 'x86_64' | |
| run: | | |
| ( cd workspace && ${{ steps.build_paths.outputs.SOURCE }}/tools/ci/scripts/macos/package_tests.sh build macos_tests_${{ matrix.build_type }} ) | |
| - name: Store the packaged tests for the x86_64 macOS-11 worker | |
| if: matrix.architecture == 'x86_64' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_tests_${{ matrix.build_type }} | |
| path: workspace/macos_tests_${{ matrix.build_type }}.tar.gz | |
| # Before we terminate this job, delete the build folder. The cache | |
| # actions will require the disk space to create the archives. | |
| - name: Reclaim disk space | |
| run: | | |
| rm -rf ${{ steps.build_paths.outputs.BINARY }} | |
| # This job takes the packaged tests (Release + Debug) from the Monterey | |
| # builder and runs them on a new Big Sur instance | |
| test_macos_bigsur: | |
| needs: build_macos | |
| runs-on: macos-11 | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: macos_tests_Release | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: macos_tests_Debug | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.2' | |
| architecture: 'x64' | |
| - name: Install tests dependencies | |
| id: install_test_deps | |
| run: | | |
| pip3 install setuptools \ | |
| pexpect==3.3 \ | |
| psutil \ | |
| timeout_decorator \ | |
| six \ | |
| thrift==0.11.0 \ | |
| osquery | |
| - name: Install CMake | |
| shell: bash | |
| run: | | |
| mkdir -p "workspace/downloads" \ | |
| "workspace/install" | |
| ./tools/ci/scripts/macos/install_cmake.sh \ | |
| "workspace/downloads" \ | |
| "workspace/install" \ | |
| "3.21.4" | |
| - name: Extract the tests | |
| run: | | |
| tar xzf macos_tests_Release.tar.gz | |
| tar xzf macos_tests_Debug.tar.gz | |
| - name: Run the Debug tests | |
| run: | | |
| ( cd macos_tests_Debug && ./run.sh ) | |
| - name: Run the Release tests | |
| run: | | |
| ( cd macos_tests_Release && ./run.sh ) | |
| # This job builds the universal macOS artifacts | |
| build_universal_macos_artifacts: | |
| needs: test_macos_bigsur | |
| runs-on: macos-12 | |
| steps: | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install CMake | |
| shell: bash | |
| run: | | |
| mkdir -p "workspace/downloads" \ | |
| "workspace/install" | |
| ./tools/ci/scripts/macos/install_cmake.sh \ | |
| "workspace/downloads" \ | |
| "workspace/install" \ | |
| "3.21.4" | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: macos_unsigned_release_package_data_x86_64 | |
| path: macos_unsigned_release_package_data_x86_64 | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: macos_unsigned_release_package_data_arm64 | |
| path: macos_unsigned_release_package_data_arm64 | |
| - name: Create the universal package data | |
| run: | | |
| tools/ci/scripts/macos/build_universal_package_data.sh | |
| - name: Store the universal unsigned release package data artifact | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_unsigned_release_package_data_universal | |
| path: package_data.tar.gz | |
| - name: Clone the osquery-packaging repository | |
| run: | | |
| git clone ${{ env.PACKAGING_REPO }} osquery-packaging | |
| cd osquery-packaging | |
| git checkout ${{ env.PACKAGING_COMMIT }} | |
| - name: Create the packages | |
| shell: bash | |
| run: | | |
| osquery_version=$(git describe --tags --always ) | |
| package_format_list=( "productbuild" "TGZ" ) | |
| for package_format in "${package_format_list[@]}" ; do | |
| cmake -DCMAKE_BUILD_TYPE=Release \ | |
| -DCPACK_GENERATOR=${package_format} \ | |
| -DOSQUERY_PACKAGE_VERSION=${osquery_version} \ | |
| -DOSQUERY_DATA_PATH=$(pwd)/universal \ | |
| -S osquery-packaging \ | |
| -B package_build | |
| cmake --build package_build \ | |
| --target package | |
| done | |
| - name: Locate the packages | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PKG_PATH=$(ls package_build/*.pkg)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_TGZ_PATH=$(ls package_build/*.tar.gz)" >> $GITHUB_OUTPUT | |
| - name: Store the PKG unsigned release packages | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_unsigned_pkg_universal | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PKG_PATH }} | |
| - name: Store the TGZ unsigned release packages | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_unsigned_tgz_universal | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_TGZ_PATH }} | |
| # The Windows build will only start once we know that the code | |
| # has been properly formatted | |
| build_windows: | |
| needs: [check_code_style, check_libraries_manifest] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| build_type: [Release] | |
| bitness: [64, arm64] | |
| os: [windows-2019] | |
| steps: | |
| - name: Select the build job count | |
| shell: powershell | |
| id: build_job_count | |
| run: | | |
| $cores=(Get-CIMInstance Win32_ComputerSystem).NumberOfLogicalProcessors + 1 | |
| echo "VALUE=$cores" >> $env:GITHUB_OUTPUT | |
| - name: Setup the build paths | |
| shell: powershell | |
| id: build_paths | |
| run: | | |
| $rel_src_path = "w\src" | |
| $rel_build_path = "w\build" | |
| $rel_sccache_path = "w\sccache" | |
| $rel_downloads_path = "w\downloads" | |
| $rel_install_path = "w\install" | |
| $rel_package_data_path = "w\package_data" | |
| $rel_packaging_path = "w\osquery-packaging" | |
| New-Item -ItemType Directory -Force -Path $rel_build_path | |
| New-Item -ItemType Directory -Force -Path $rel_sccache_path | |
| New-Item -ItemType Directory -Force -Path $rel_downloads_path | |
| New-Item -ItemType Directory -Force -Path $rel_install_path | |
| New-Item -ItemType Directory -Force -Path $rel_package_data_path | |
| $base_dir = (Get-Item .).FullName | |
| echo "SOURCE=$base_dir\$rel_src_path" >> $env:GITHUB_OUTPUT | |
| echo "REL_SOURCE=$rel_src_path" >> $env:GITHUB_OUTPUT | |
| echo "BINARY=$base_dir\$rel_build_path" >> $env:GITHUB_OUTPUT | |
| echo "SCCACHE=$base_dir\$rel_sccache_path" >> $env:GITHUB_OUTPUT | |
| echo "DOWNLOADS=$base_dir\$rel_downloads_path" >> $env:GITHUB_OUTPUT | |
| echo "INSTALL=$base_dir\$rel_install_path" >> $env:GITHUB_OUTPUT | |
| echo "PACKAGING=$base_dir\$rel_packaging_path" >> $env:GITHUB_OUTPUT | |
| echo "PACKAGE_DATA=$base_dir\$rel_package_data_path" >> $env:GITHUB_OUTPUT | |
| - name: Setup the VC arch | |
| shell: powershell | |
| id: vc_arch | |
| run: | | |
| $arch = switch ("${{ matrix.bitness }}") | |
| { | |
| "64" { "x64" } | |
| "arm64" { "x64_arm64" } | |
| } | |
| echo "VC_ARCH=$arch" >> $env:GITHUB_OUTPUT | |
| # Symbolic links are supported by default on Linux and macOS. On | |
| # Windows, we have to enable them explicitly. They are used to | |
| # handle the include header namespace support that came with | |
| # the initial Buck build system refactor | |
| - name: Configure git | |
| run: | | |
| git config --global core.autocrlf false | |
| git config --global core.symlinks true | |
| - name: Clone the osquery repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| path: ${{ steps.build_paths.outputs.REL_SOURCE }} | |
| - name: Get runner hardware info | |
| id: runner_stats | |
| shell: powershell | |
| run: | | |
| Get-CIMInstance Win32_LogicalDisk | |
| Get-CIMInstance Win32_ComputerSystem | Select-Object -Property NumberOfLogicalProcessors | |
| Get-CIMInstance Win32_ComputerSystem | Select-Object -Property TotalPhysicalMemory | |
| - name: Detect the osquery version | |
| shell: powershell | |
| id: osquery_version | |
| run: | | |
| cd ${{ steps.build_paths.outputs.SOURCE }} | |
| $osquery_version=$(git describe --tags --abbrev=0) | |
| echo "VALUE=$osquery_version" >> $env:GITHUB_OUTPUT | |
| - name: Clone the osquery-packaging repository | |
| run: | | |
| cd w | |
| git clone ${{ env.PACKAGING_REPO }} | |
| cd ${{ steps.build_paths.outputs.PACKAGING }} | |
| git checkout ${{ env.PACKAGING_COMMIT }} | |
| - name: Update the cache (git submodules) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SOURCE }}\.git\modules | |
| key: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}}_${{ github.sha }} | |
| restore-keys: | | |
| gitmodules_${{ matrix.os }}_${{env.SUBMODULE_CACHE_VERSION}} | |
| - name: Update the cache (downloads) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.DOWNLOADS }} | |
| key: | | |
| downloads_${{ matrix.os }}_${{ github.sha }} | |
| restore-keys: | | |
| downloads_${{ matrix.os }} | |
| - name: Initialize the Python 3 installation | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.x" | |
| architecture: "x64" | |
| # The runners will likely have both the x86 and x64 versions of | |
| # Python but we always need the 64-bit one regardless of which | |
| # architecture we are building for. | |
| # | |
| # The setup-python action should have put the right Python version | |
| # in the PATH variable for us, so locate the installation directory | |
| # so we can use it as a hint when we configure the project with | |
| # CMake | |
| - name: Locate the Python root directory | |
| id: python_root_directory | |
| shell: powershell | |
| run: | | |
| $python_executable_path = $(Get-Command python.exe | Select-Object -ExpandProperty Definition) | |
| $python_root_directory = (Get-Item $python_executable_path).Directory.FullName | |
| echo "VALUE=$python_root_directory" >> $env:GITHUB_OUTPUT | |
| # Install the Python dependencies needed for our testing framework | |
| - name: Install tests prerequisites | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel ` | |
| setuptools ` | |
| psutil ` | |
| timeout_decorator ` | |
| thrift==0.11.0 ` | |
| osquery ` | |
| pywin32 | |
| - name: Install Strawberry Perl | |
| working-directory: ${{ steps.build_paths.outputs.SOURCE }} | |
| shell: powershell | |
| run: | | |
| tools\ci\scripts\install_openssl_formula_dependencies.ps1 | |
| # TODO: Implement a cache so we don't keep hitting the server at each run | |
| - name: Install CMake | |
| working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} | |
| shell: powershell | |
| run: | | |
| $long_cmake_ver = "3.21.4" | |
| $short_cmake_ver = $($long_cmake_ver.split(".")[0] + "." + $long_cmake_ver.split(".")[1]) | |
| $folder_name = $("cmake-" + $long_cmake_ver + "-windows-x86_64") | |
| $archive_name = $($folder_name + ".zip") | |
| $url = $("https://cmake.org/files/v" + $short_cmake_ver + "/" + $archive_name) | |
| (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) | |
| 7z x -o${{ steps.build_paths.outputs.INSTALL }} -y $archive_name | |
| echo "${{ steps.build_paths.outputs.INSTALL }}\$folder_name\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| # TODO: Implement a cache so we don't keep hitting the server at each run | |
| - name: Install sccache | |
| working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} | |
| shell: powershell | |
| run: | | |
| $long_version = "0.0.1" | |
| $archive_name = $("sccache-" + $long_version + "-windows.7z") | |
| $url = $("https://github.com/osquery/sccache/releases/download/" + $long_version + "-osquery/" + $archive_name) | |
| (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) | |
| 7z x -o${{ steps.build_paths.outputs.INSTALL }}\sccache -y $archive_name | |
| echo "${{ steps.build_paths.outputs.INSTALL }}\sccache" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| # TODO: Implement a cache so we don't keep hitting the server at each run | |
| - name: Install Ninja | |
| working-directory: ${{ steps.build_paths.outputs.DOWNLOADS }} | |
| shell: powershell | |
| run: | | |
| $long_version = "1.10.0" | |
| $archive_name = "ninja-win.zip" | |
| $url = $("https://github.com/ninja-build/ninja/releases/download/v" + $long_version + "/" + $archive_name) | |
| (New-Object System.Net.WebClient).DownloadFile($url, $archive_name) | |
| 7z x -o${{ steps.build_paths.outputs.INSTALL }}\ninja -y $archive_name | |
| echo "${{ steps.build_paths.outputs.INSTALL }}\ninja" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| - name: Configure the project | |
| shell: cmd | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} | |
| SCCACHE_CACHE_SIZE: "5G" | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} | |
| @echo on | |
| set cross_compilation= | |
| if ${{ matrix.bitness }}==arm64 ( | |
| echo set (CMAKE_SYSTEM_NAME Windows^) > cross.cmake | |
| echo set (CMAKE_SYSTEM_PROCESSOR ARM64^) >> cross.cmake | |
| set cross_compilation=-DCMAKE_TOOLCHAIN_FILE=cross.cmake | |
| ) | |
| cmake -G Ninja ^ | |
| -DCMAKE_C_COMPILER=cl.exe ^ | |
| -DCMAKE_CXX_COMPILER=cl.exe ^ | |
| -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ | |
| -DOSQUERY_BUILD_TESTS=ON ^ | |
| -DCMAKE_C_COMPILER_LAUNCHER="sccache.exe" ^ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER="sccache.exe" ^ | |
| -DPython3_ROOT_DIR=${{ steps.python_root_directory.outputs.VALUE }} ^ | |
| ${{ steps.build_paths.outputs.SOURCE }} ^ | |
| %cross_compilation% | |
| - name: Determine compiler version | |
| id: determine_compiler_version | |
| shell: pwsh | |
| run: | | |
| $compiler = (Get-Content "${{ steps.build_paths.outputs.BINARY }}\CMakeCache.txt" | Select-String -Pattern "CMAKE_CXX_COMPILER:STRING=(.*)").Matches[0].Groups[1].Value | |
| echo "Compiler configured by CMake is $compiler" | |
| if ($compiler -eq $null || $compiler -eq "") { | |
| Write-Error "Could not find the configured compiler" -ErrorAction Stop | |
| } | |
| <# | |
| We run the compiler help option; the compiler will write its version in stderr. | |
| Due to how powershell works, we have to go through some hoops to extract the stderr to a variable | |
| and also avoid it considering the command as failed because stderr contains messages. | |
| The expression runs the compiler in a subshell, discards its stdout, then the stderr of the subshell is redirected | |
| to the stdout of the parent shell. | |
| #> | |
| $ErrorActionPreference = 'Continue' | |
| $erroutput = $( & "$compiler" /? 1>$null ) 2>&1 | |
| $ErrorActionPreference = 'Stop' | |
| if ($erroutput -eq $null || $erroutput -eq "") { | |
| Write-Error "Failed to run the compiler at $compiler" -ErrorAction Stop | |
| } | |
| $version = ($erroutput | Select-String -Pattern "Compiler Version (.*) for").Matches[0].Groups[1].Value.Replace(".", "") | |
| if ($version -eq $null || $version -eq "") { | |
| Write-Error "Failed to determine compiler version for $compiler and output $erroutput" -ErrorAction Stop | |
| } | |
| echo "Found compiler version $version" | |
| echo "COMPILER_VERSION=$version" >> $env:GITHUB_OUTPUT | |
| - name: Update the cache (sccache) | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ steps.build_paths.outputs.SCCACHE }} | |
| key: | | |
| sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }}_${{ github.sha }} | |
| restore-keys: | | |
| sccache_${{ matrix.os }}_${{ matrix.bitness }}_${{ matrix.build_type }}_${{ steps.determine_compiler_version.outputs.COMPILER_VERSION }} | |
| - name: Build the project | |
| shell: cmd | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| SCCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} | |
| SCCACHE_CACHE_SIZE: "5G" | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} | |
| @echo on | |
| cmake --build . -j ${{ steps.build_job_count.outputs.VALUE }} | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| sccache.exe --stop-server | |
| - name: Disk space information | |
| id: disk_space_info_post_build | |
| shell: powershell | |
| run: | | |
| Get-CIMInstance Win32_LogicalDisk | |
| $bindir_size = (Get-ChildItem -Path ${{ steps.build_paths.outputs.BINARY }} -Recurse | Measure-Object -Property Length -Sum).sum / 1MB | |
| echo "Binary dir size: $bindir_size MBs" | |
| - name: Run the tests | |
| # arm64 tests can't be run on x64 machine | |
| if: matrix.bitness != 'arm64' | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| shell: cmd | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} | |
| @echo on | |
| ctest --build-nocmake -C Release -V | |
| - name: Run the install target | |
| working-directory: ${{ steps.build_paths.outputs.BINARY }} | |
| env: | |
| CCACHE_DIR: ${{ steps.build_paths.outputs.SCCACHE }} | |
| DESTDIR: ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| run: | | |
| cmake --build . --target install -j 3 | |
| - name: Create the packages | |
| shell: cmd | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ steps.vc_arch.outputs.VC_ARCH }} | |
| @echo on | |
| cd ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| 7z ^ | |
| a windows_package_data.zip ^ | |
| ${{ steps.build_paths.outputs.PACKAGE_DATA }} | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| if ${{ matrix.bitness }}==arm64 ( | |
| echo Do not create other packages for arm64 | |
| exit 0 | |
| ) | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ | |
| -DCPACK_GENERATOR=WIX ^ | |
| -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ | |
| -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ | |
| -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| cmake --build . ^ | |
| --config Release ^ | |
| --target package | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ^ | |
| -DCPACK_GENERATOR=NuGet ^ | |
| -DOSQUERY_PACKAGE_VERSION=${{ steps.osquery_version.outputs.VALUE }} ^ | |
| -DOSQUERY_DATA_PATH=${{ steps.build_paths.outputs.PACKAGE_DATA }} ^ | |
| -DOSQUERY_BITNESS=${{ matrix.bitness }} ^ | |
| ${{ steps.build_paths.outputs.PACKAGING }} | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| cmake --build . ^ | |
| --config Release ^ | |
| --target package | |
| if %errorlevel% neq 0 exit /b %errorlevel% | |
| - name: Locate the packages | |
| working-directory: ${{ steps.build_paths.outputs.PACKAGE_BUILD }} | |
| id: packages | |
| shell: bash | |
| run: | | |
| echo "REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH=$(ls *.zip)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_MSI_PATH=$(ls *.msi)" >> $GITHUB_OUTPUT | |
| echo "REL_UNSIGNED_RELEASE_NUPKG_PATH=$(ls *.nupkg)" >> $GITHUB_OUTPUT | |
| - name: Store the unsigned release package data artifact | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: windows${{ matrix.bitness }}_unsigned_release_package_data | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_PACKAGE_DATA_PATH }} | |
| - name: Store the unsigned release MSI artifact | |
| if: matrix.bitness != 'arm64' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: windows${{ matrix.bitness }}_unsigned_release_msi | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_MSI_PATH }} | |
| - name: Store the unsigned release NUPKG artifact | |
| if: matrix.bitness != 'arm64' | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: windows${{ matrix.bitness }}_unsigned_release_nupkg | |
| path: ${{ steps.packages.outputs.REL_UNSIGNED_RELEASE_NUPKG_PATH }} | |
| # Before we terminate this job, delete the build folder. The cache | |
| # actions will require the disk space to create the archives. | |
| - name: Reclaim disk space | |
| shell: powershell | |
| run: | | |
| rm -r -Force ${{ steps.build_paths.outputs.BINARY }} | |
| # This job is here as a github status check -- it allows us to move the dependency from | |
| # being on all the jobs to this single one. | |
| mergeable: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: true | |
| needs: | |
| - build_macos | |
| - build_linux | |
| - build_windows | |
| - build_universal_macos_artifacts | |
| - check_source_code |