Skip to content
Rebuild Native C++ Libraries and Push to PR

Rebuild Native C++ Libraries and Push to PR #107

Sign in to view logs

Rebuild Native C++ Libraries and Push to PR

Rebuild Native C++ Libraries and Push to PR #107

Workflow file for this run

.github/workflows/rebuild_native_libs.yml at bdbac44
name: Rebuild Native C++ Libraries and Push to PR
on:
workflow_dispatch:
# This workflow is triggered manually from the Actions tab.
# It's meant to be run on a PR that changes the native C++ code.
# It builds native libraries for all supported platforms and pushes them to the current branch.
# It splits the building process into 3 build jobs:
# 1. build-freebsd - Builds native libraries for x64 FreeBSD
# 2. build-macos - Builds native libraries for MacOS: both ARM and x64.
# It uses native runners for each platform, because cross compilation on MacOS is complicated.
# 3. build-others - Builds native libraries for x64 Linux, ARM Linux and Windows.
# It uses cross-compilation for ARM Linux and Windows.
#
# Each build job saves the resulting binaries to the cache under a unique key
# When all build jobs are finished, the collect-commit-and-push job restores the binaries from the cache
# and pushes them to the current branch.
jobs:
build-rust-windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
- name: Build Rust Library
run: |
cd core/rust/qdbr
cargo build --release
- name: log git status
run: |
git status
- name: Copy Rust Library to the final directory
run: |
cp core/rust/qdbr/target/release/questdbr.dll core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
- name: Save Windows Rust Library to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
key: nativelibs-windows-rust-${{ github.sha }}
build-all-freebsd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build in FreeBSD
# @d44bf830e7d is the same as the tag v1, but it's guaranteed to be immutable.
uses: vmactions/freebsd-vm@d44bf830e7d850111d27aa120d468c905e5980f4
with:
# FreeBSD 13.2
release: 13.2
usesh: true
prepare: |
pkg install -y curl cmake gcc git nasm openjdk17 rust
run: |
export JAVA_HOME=/usr/local/openjdk17
cd core
cmake -B build/release -DCMAKE_BUILD_TYPE=Release
cmake --build build/release --config Release
cd rust/qdbr
cargo build --release
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/freebsd/
- name: log git status
run: |
git status
- name: Save FreeBSD Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
key: nativelibs-freebsd-${{ github.sha }}
build-all-macos:
strategy:
matrix:
# macos-14 = ARM M1
# macos-latest = x64
# if you change OS definitions then you need to change conditions in cache-save steps below
os: [ macos-14, macos-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Install toolchains (CXX/NASM)
run: |
brew install make cmake gcc nasm
- name: Build CXX Library
run: |
cd core
cmake -B build/release -DCMAKE_BUILD_TYPE=Release
cmake --build build/release --config Release
- name: Build Rust Library
run: |
cd core/rust/qdbr
cargo clean
cargo build --release
- name: Copy darwin-aarch64 Rust Library to the final directory
if: ${{ matrix.os == 'macos-14' }}
run: |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/armosx/
- name: Copy darwin-amd64 Rust Library to the final directory
if: ${{ matrix.os == 'macos-latest' }}
run: |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/osx/
- name: Save darwin-aarch64 Libraries to Cache
if: ${{ matrix.os == 'macos-14' }}
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
key: nativelibs-armosx-${{ github.sha }}
- name: Save darwin-amd64 Libraries to Cache
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
key: nativelibs-osx-${{ github.sha }}
build-all-linux-amd64:
runs-on: ubuntu-latest
# this is very bare container, we use it to restrict GLIBC to 2.28
# everything has to be installed on it manually
container:
image: debian:buster
steps:
- uses: actions/checkout@v4
- name: Install toolchains (CXX/NASM/Rust)
run: |
ldd --version
env
apt-get update -y
apt install git nasm python3 wget ca-certificates build-essential zstd -y
python3 ./.github/prepare_rust_env.py
echo "PATH=/github/home/.cargo/bin/:$PATH" >> "$GITHUB_ENV"
- name: Install up-to-date CMake
run: |
wget -nv https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz
tar -zxf cmake-3.29.2-linux-x86_64.tar.gz
echo "PATH=`pwd`/cmake-3.29.2-linux-x86_64/bin/:$PATH" >> "$GITHUB_ENV"
- name: Install JDK17 (for jni.h)
run: |
wget -nv https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
tar xfz OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
echo "JAVA_HOME=`pwd`/jdk-17.0.11+9/" >> "$GITHUB_ENV"
- name: Generate Makefiles
run: |
cd ./core
cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: Build linux-amd64 CXX Library
run: |
cd core
cmake --build cmake-build-release --config Release
- name: Build linux-amd64 Rust Library
run: |
cd core/rust/qdbr
cargo clean
cargo build --release
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/linux/
- name: Save linux-amd64 Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
key: nativelibs-linux-${{ github.sha }}
build-all-linux-aarch64:
runs-on: ubuntu-latest
env:
generator: "CodeBlocks - Unix Makefiles"
steps:
- uses: actions/checkout@v4
- name: Increase file count and install tooling
run: |
env
sudo sysctl -w fs.file-max=500000
sudo apt-get update -y
sudo apt-get install -y nasm g++-10-aarch64-linux-gnu gcc-10-aarch64-linux-gnu
- name: Install rs-cross
run: |
cargo install cross
- name: Generate Makefiles
run: |
cd ./core
cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: CMAKE linux-aarch64
run: |
cd ./core
cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake -DARM_TOOLCHAIN_PATH=/usr/bin/ -DCMAKE_EXECUTABLE_SUFFIX=-10 -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-arm64
- name: Build linux-aarch64 CXX Library
run: |
cd core
cmake --build cmake-build-release-arm64 --config Release
- name: Build linux-aarch64 Rust Library
run: |
cd core/rust/qdbr
cargo clean
cross build --target aarch64-unknown-linux-gnu --release --target-dir `pwd`/target
cp target/aarch64-unknown-linux-gnu/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/armlinux/
- name: Check git status
run: |
git status
- name: Save linux-aarch64 Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
key: nativelibs-armlinux-${{ github.sha }}
build-cxx-windows:
runs-on: ubuntu-latest
env:
generator: "CodeBlocks - Unix Makefiles"
steps:
- uses: actions/checkout@v4
- name: Increase file count and install tooling
run: |
env
sudo sysctl -w fs.file-max=500000
sudo apt-get update -y
sudo apt-get install -y nasm gcc-mingw-w64 g++-mingw-w64
- name: Generate Makefiles
run: |
cd ./core
cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: Download windows jni_md.h from JDK 11
run: |
cd core
curl https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-jdk11/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h
- name: CMake Windows
run: |
cd core
cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/windows-x86_64.cmake -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-win64
- name: Build Windows C/C++ Library
run: |
cd core
cmake --build cmake-build-release-win64 --config Release
- name: Check C++ runtime dependency
run: |
cd ./core
if x86_64-w64-mingw32-objdump -p ./src/main/resources/io/questdb/bin/windows/libquestdb.dll | grep -q libstdc++; then
echo "Failure: C++ runtime dependency detected"
exit 1
fi
- name: Check git status
run: |
git status
- name: Save Windows C/C++ Library to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
key: nativelibs-windows-${{ github.sha }}
collect-commit-and-push:
needs: [ build-all-freebsd, build-all-macos,build-cxx-windows, build-rust-windows, build-all-linux-amd64, build-all-linux-aarch64 ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Print file sizes before
run: |
pwd
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
- name: Restore FreeBSD Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
key: nativelibs-freebsd-${{ github.sha }}
- name: Restore darwin-aarch64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
key: nativelibs-armosx-${{ github.sha }}
- name: Restore darwin-amd64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
key: nativelibs-osx-${{ github.sha }}
- name: Restore linux-amd64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
key: nativelibs-linux-${{ github.sha }}
- name: Restore linux-aarch64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
key: nativelibs-armlinux-${{ github.sha }}
- name: Restore Windows CXX Library from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
key: nativelibs-windows-${{ github.sha }}
- name: Restore Windows Rust Library from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
key: nativelibs-windows-rust-${{ github.sha }}
- name: Commit the files
run: |
git config --global user.name 'GitHub Actions - Rebuild Native Libraries'
git config --global user.email 'jaromir@questdb.io'
git add core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
git add core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
git add core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
git add core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/linux/libquestdb.so
git add core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
git add core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
git commit -m "chore(build): Rebuild Native Libraries"
- name: Check git status
run: |
git status
- name: Print file sizes after
run: |
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
- name: Push the files to the current branch
uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df
# Why do we use a commit hash instead of a tag for the github-push-action?
# ad-m/github-push-action is not as well-known repo as e.g. actions/checkout, and therefore we trust it less.
# d91a48109 is the same as the tag v0.8.0, but it's guaranteed to be immutable.
# So even if a bad actor takes over the repo, and rewrites tags to point to malicious commits, we will still be safe.
with:
branch: ${{ github.head_ref || github.ref_name }}