Rebuild Native C++ Libraries and Push to PR #73
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Rebuild Native C++ Libraries and Push to PR | |
on: | |
workflow_dispatch: | |
# This workflow is triggered manually from the Actions tab. | |
# It's meant to be run on a PR that changes the native C++ code. | |
# It builds native libraries for all supported platforms and pushes them to the current branch. | |
# It splits the building process into 3 build jobs: | |
# 1. build-freebsd - Builds native libraries for x64 FreeBSD | |
# 2. build-macos - Builds native libraries for MacOS: both ARM and x64. | |
# It uses native runners for each platform, because cross compilation on MacOS is complicated. | |
# 3. build-others - Builds native libraries for x64 Linux, ARM Linux and Windows. | |
# It uses cross-compilation for ARM Linux and Windows. | |
# | |
# Each build job saves the resulting binaries to the cache under a unique key | |
# When all build jobs are finished, the collect-commit-and-push job restores the binaries from the cache | |
# and pushes them to the current branch. | |
jobs: | |
build-windows: | |
runs-on: windows-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Build Rust Library | |
run: | | |
cd core/rust/qdbr | |
cargo build --release | |
- name: log git status | |
run: | | |
git status | |
- name: Copy Rust Library to the final directory | |
run: | | |
cp core/rust/qdbr/target/release/questdbr.dll core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll | |
- name: Save Windows Rust Library to Cache | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll | |
key: nativelibs-windows-rust-${{ github.sha }} | |
build-freebsd: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Build in FreeBSD | |
# @d44bf830e7d is the same as the tag v1, but it's guaranteed to be immutable. | |
uses: vmactions/freebsd-vm@d44bf830e7d850111d27aa120d468c905e5980f4 | |
with: | |
# FreeBSD 13.2 | |
release: 13.2 | |
usesh: true | |
prepare: | | |
pkg install -y curl cmake gcc git nasm openjdk17 rust | |
run: | | |
export JAVA_HOME=/usr/local/openjdk17 | |
cd core | |
cmake -B build/release -DCMAKE_BUILD_TYPE=Release | |
cmake --build build/release --config Release | |
cd rust/qdbr | |
cargo build --release | |
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/freebsd/ | |
- name: log git status | |
run: | | |
git status | |
- name: Save FreeBSD Libraries to Cache | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so | |
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so | |
key: nativelibs-freebsd-${{ github.sha }} | |
build-macos: | |
strategy: | |
matrix: | |
# macos-14 = ARM M1 | |
# macos-latest = x64 | |
# if you change OS definitions then you need to change conditions in cache-save steps below | |
os: [ macos-14, macos-latest ] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install tooling | |
run: | | |
brew install make cmake gcc nasm | |
- name: Build MacOS C/C++ Library | |
run: | | |
cd core | |
cmake -B build/release -DCMAKE_BUILD_TYPE=Release | |
cmake --build build/release --config Release | |
- name: Build MacOS Rust Library | |
run: | | |
cd core/rust/qdbr | |
cargo clean | |
cargo build --release | |
- name: Copy ARM MacOS Rust Library to the final directory | |
if: ${{ matrix.os == 'macos-14' }} | |
run: | | |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/armosx/ | |
- name: Copy x64 MacOS Rust Library to the final directory | |
if: ${{ matrix.os == 'macos-latest' }} | |
run: | | |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/osx/ | |
- name: Save ARM MacOS Libraries to Cache | |
if: ${{ matrix.os == 'macos-14' }} | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib | |
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib | |
key: nativelibs-armosx-${{ github.sha }} | |
- name: Save x64 MacOS Libraries to Cache | |
if: ${{ matrix.os == 'macos-latest' }} | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib | |
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib | |
key: nativelibs-osx-${{ github.sha }} | |
build-others: | |
runs-on: ubuntu-latest | |
container: debian:buster | |
env: | |
generator: "CodeBlocks - Unix Makefiles" | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install toolchains (C/C++/NASM/Rust) | |
run: | | |
apt-get update -y | |
apt install nasm python3 wget ca-certificates cmake build-essential -y | |
python3 ./.github/prepare_rust_env.py | |
export PATH=$HOME/.cargo/bin:$PATH | |
export CARGO_HOME=$HOME/.cargo | |
- name: Generate Makefiles | |
run: | | |
cd ./core | |
cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S. | |
- name: Build x64 Linux C/C++ Library | |
run: | | |
cd core | |
cmake --build cmake-build-release --config Release | |
- name: Build x64 Linux Rust Library | |
run: | | |
cd core/rust/qdbr | |
cargo clean | |
cargo build --release | |
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/linux/ | |
# - name: CMAKE ARM Linux | |
# run: | | |
# cd ./core | |
# cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake -DARM_TOOLCHAIN_PATH=/usr/bin/ -DCMAKE_EXECUTABLE_SUFFIX=-10 -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-arm64 | |
# - name: Build ARM Linux C/C++ Library | |
# run: | | |
# cd core | |
# cmake --build cmake-build-release-arm64 --config Release | |
# - name: Build ARM Linux Rust Library | |
# run: | | |
# cd core/rust/qdbr | |
# cargo clean | |
# cross build --target aarch64-unknown-linux-gnu --release --target-dir `pwd`/target | |
# cp target/aarch64-unknown-linux-gnu/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/armlinux/ | |
- name: Download windows jni_md.h from JDK 11 | |
run: | | |
cd core | |
curl https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-jdk11/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h | |
- name: CMake Windows | |
run: | | |
cd core | |
cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/windows-x86_64.cmake -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-win64 | |
- name: Build Windows C/C++ Library | |
run: | | |
cd core | |
cmake --build cmake-build-release-win64 --config Release | |
- name: Check C++ runtime dependency | |
run: | | |
cd ./core | |
if x86_64-w64-mingw32-objdump -p ./src/main/resources/io/questdb/bin/windows/libquestdb.dll | grep -q libstdc++; then | |
echo "Failure: C++ runtime dependency detected" | |
exit 1 | |
fi | |
- name: Check git status | |
run: | | |
git status | |
- name: Save x64 Linux Libraries to Cache | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so | |
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so | |
key: nativelibs-linux-${{ github.sha }} | |
- name: Save ARM Linux Libraries to Cache | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so | |
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so | |
key: nativelibs-armlinux-${{ github.sha }} | |
- name: Save Windows C/C++ Library to Cache | |
uses: actions/cache/save@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll | |
key: nativelibs-windows-${{ github.sha }} | |
collect-commit-and-push: | |
needs: [ build-freebsd, build-macos, build-others, build-windows ] | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Print file sizes before | |
run: | | |
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \; | |
- name: Restore FreeBSD Libraries from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so | |
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so | |
key: nativelibs-freebsd-${{ github.sha }} | |
- name: Restore ARM MacOS Libraries from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib | |
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib | |
key: nativelibs-armosx-${{ github.sha }} | |
- name: Restore x64 MacOS Libraries from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib | |
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib | |
key: nativelibs-osx-${{ github.sha }} | |
- name: Restore x64 Linux Libraries from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so | |
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so | |
key: nativelibs-linux-${{ github.sha }} | |
- name: Restore ARM Linux Libraries from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so | |
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so | |
key: nativelibs-armlinux-${{ github.sha }} | |
- name: Restore Windows C/C++ Library from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll | |
key: nativelibs-windows-${{ github.sha }} | |
- name: Restore Windows Rust Library from Cache | |
uses: actions/cache/restore@v3 | |
with: | |
path: | | |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll | |
key: nativelibs-windows-rust-${{ github.sha }} | |
- name: Commit the files | |
run: | | |
git config --global user.name 'GitHub Actions - Rebuild Native Libraries' | |
git config --global user.email 'jaromir@questdb.io' | |
git add core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib | |
git add core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib | |
git add core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib | |
git add core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib | |
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so | |
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so | |
git add core/src/main/resources/io/questdb/bin/linux/libquestdb.so | |
git add core/src/main/resources/io/questdb/bin/linux/libquestdbr.so | |
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so | |
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so | |
git add core/src/main/resources/io/questdb/bin/windows/libquestdb.dll | |
git add core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll | |
git commit -m "chore(build): Rebuild Native Libraries" | |
- name: Check git status | |
run: | | |
git status | |
- name: Print file sizes after | |
run: | | |
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \; | |
- name: Push the files to the current branch | |
uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df | |
# Why do we use a commit hash instead of a tag for the github-push-action? | |
# ad-m/github-push-action is not as well-known repo as e.g. actions/checkout and therefore we trust it less. | |
# d91a48109 is the same as the tag v0.8.0, but it's guaranteed to be immutable. | |
# So even if a bad actor takes over the repo, and rewrites tags to point to malicious commits, we will still be safe. | |
with: | |
branch: ${{ github.head_ref || github.ref_name }} | |