rebuild_native_libs.yml

name: Rebuild Native C++ Libraries and Push to PR
on:
  workflow_dispatch:
# This workflow is triggered manually from the Actions tab.
# It's meant to be run on a PR that changes the native C++ code.

# It builds native libraries for all supported platforms and pushes them to the current branch.
# It splits the building process into 3 build jobs:
# 1. build-freebsd - Builds native libraries for x64 FreeBSD
# 2. build-macos - Builds native libraries for MacOS: both ARM and x64.
#    It uses native runners for each platform, because cross compilation on MacOS is complicated.
# 3. build-others - Builds native libraries for x64 Linux, ARM Linux and Windows.
#    It uses cross-compilation for ARM Linux and Windows.
#
# Each build job saves the resulting binaries to the cache under a unique key
# When all build jobs are finished, the collect-commit-and-push job restores the binaries from the cache
# and pushes them to the current branch.

jobs:
  build-windows:
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
      - name: Build Rust Library
        run: |
          cd core/rust/qdbr
          cargo build --release
      - name: log git status
        run: |
          git status
      - name: Copy Rust Library to the final directory
        run: |
          cp core/rust/qdbr/target/release/questdbr.dll core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
      - name: Save Windows Rust Library to Cache
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
          key: nativelibs-windows-rust-${{ github.sha }}
  build-freebsd:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Build in FreeBSD
        # @d44bf830e7d is the same as the tag v1, but it's guaranteed to be immutable.
        uses: vmactions/freebsd-vm@d44bf830e7d850111d27aa120d468c905e5980f4
        with:
          # FreeBSD 13.2
          release: 13.2
          usesh: true
          prepare: |
            pkg install -y curl cmake gcc git nasm openjdk17 rust
          run: |
            export JAVA_HOME=/usr/local/openjdk17
            cd core
            cmake -B build/release -DCMAKE_BUILD_TYPE=Release
            cmake --build build/release --config Release
            cd rust/qdbr
            cargo build --release
            cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/freebsd/
      - name: log git status
        run: |
          git status
      - name: Save FreeBSD Libraries to Cache
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
            core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
          key: nativelibs-freebsd-${{ github.sha }}
  build-macos:
    strategy:
      matrix:
        # macos-14 = ARM M1
        # macos-latest = x64
        # if you change OS definitions then you need to change conditions in cache-save steps below
        os: [ macos-14, macos-latest ]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Install toolchains (CXX/NASM)
        run: |
          brew install make cmake gcc nasm
      - name: Build CXX Library
        run: |
          cd core
          cmake -B build/release -DCMAKE_BUILD_TYPE=Release
          cmake --build build/release --config Release
      - name: Build Rust Library
        run: |
          cd core/rust/qdbr
          cargo clean
          cargo build --release
      - name: Copy darwin-aarch64 Rust Library to the final directory
        if: ${{ matrix.os == 'macos-14' }}
        run: |
          cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/armosx/
      - name: Copy darwin-amd64 Rust Library to the final directory
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/osx/
      - name: Save darwin-aarch64 Libraries to Cache
        if: ${{ matrix.os == 'macos-14' }}
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
            core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
          key: nativelibs-armosx-${{ github.sha }}
      - name: Save darwin-amd64 Libraries to Cache
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
            core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
          key: nativelibs-osx-${{ github.sha }}
  build-linux-amd64:
    runs-on: ubuntu-latest
    # this is very bare container, we use it to restrict GLIBC to 2.28
    # everything has to be installed on it manually
    container:
      image: debian:buster
    steps:
      - uses: actions/checkout@v4
      - name: Install toolchains (CXX/NASM/Rust)
        run: |
          ldd --version
          env
          apt-get update -y
          apt install git nasm python3 wget ca-certificates build-essential zstd-y
          python3 ./.github/prepare_rust_env.py
          echo "PATH=/github/home/.cargo/bin/:$PATH" >> "$GITHUB_ENV"
      - name: Install up-to-date CMake
        run: |
          wget -nv https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz
          tar -zxf cmake-3.29.2-linux-x86_64.tar.gz
          echo "PATH=`pwd`/cmake-3.29.2-linux-x86_64/bin/:$PATH" >> "$GITHUB_ENV"
      - name: Install JDK17 (for jni.h)
        run: |
          wget -nv https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
          tar xfz OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
          echo "JAVA_HOME=`pwd`/jdk-17.0.11+9/" >> "$GITHUB_ENV"
      - name: Generate Makefiles
        run: |
          cd ./core
          cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
      - name: Build linux-amd64 CXX Library
        run: |
          cd core
          cmake --build cmake-build-release --config Release
      - name: Build linux-amd64 Rust Library
        run: |
          cd core/rust/qdbr
          cargo clean
          cargo build --release
          cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/linux/
      - name: Save linux-amd64 Libraries to Cache
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/linux/libquestdb.so
            core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
          key: nativelibs-linux-${{ github.sha }}
  build-others:
    runs-on: ubuntu-20.04
    env:
      generator: "CodeBlocks - Unix Makefiles"
    steps:
      - uses: actions/checkout@v4
      - name: Increase file count and install tooling
        run: |
          env
          sudo sysctl -w fs.file-max=500000
          sudo apt-get update -y
          sudo apt-get install -y nasm g++-10-aarch64-linux-gnu gcc-10-aarch64-linux-gnu gcc-mingw-w64 g++-mingw-w64
      - name: Install rs-cross
        run: |
          cargo install cross
      - name: Generate Makefiles
        run: |
          cd ./core
          cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
      - name: CMAKE linux-aarch64
        run: |
          cd ./core
          cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake -DARM_TOOLCHAIN_PATH=/usr/bin/ -DCMAKE_EXECUTABLE_SUFFIX=-10 -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-arm64
      - name: Build linux-aarch64 CXX Library
        run: |
          cd core
          cmake --build cmake-build-release-arm64 --config Release
      - name: Build linux-aarch64 Rust Library
        run: |
          cd core/rust/qdbr
          cargo clean
          cross build --target aarch64-unknown-linux-gnu --release --target-dir `pwd`/target
          cp target/aarch64-unknown-linux-gnu/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/armlinux/
      - name: Download windows jni_md.h from JDK 11
        run: |
          cd core
          curl https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-jdk11/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h
      - name: CMake Windows
        run: |
          cd core
          cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/windows-x86_64.cmake -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-win64
      - name: Build Windows C/C++ Library
        run: |
          cd core
          cmake --build cmake-build-release-win64 --config Release
      - name: Check C++ runtime dependency
        run: |
          cd ./core
          if x86_64-w64-mingw32-objdump -p ./src/main/resources/io/questdb/bin/windows/libquestdb.dll | grep -q libstdc++; then
            echo "Failure: C++ runtime dependency detected"
            exit 1
          fi
      - name: Check git status
        run: |
          git status
      - name: Save linux-aarch64 Libraries to Cache
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
            core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
          key: nativelibs-armlinux-${{ github.sha }}
      - name: Save Windows C/C++ Library to Cache
        uses: actions/cache/save@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
          key: nativelibs-windows-${{ github.sha }}
  collect-commit-and-push:
    needs: [ build-freebsd, build-macos, build-others, build-windows, build-linux-amd64 ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Print file sizes before
        run: |
          pwd
          find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
      - name: Restore FreeBSD Libraries from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
            core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
          key: nativelibs-freebsd-${{ github.sha }}
      - name: Restore darwin-aarch64 Libraries from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
            core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
          key: nativelibs-armosx-${{ github.sha }}
      - name: Restore darwin-amd64 Libraries from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
            core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
          key: nativelibs-osx-${{ github.sha }}
      - name: Restore linux-amd64 Libraries from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/linux/libquestdb.so
            core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
          key: nativelibs-linux-${{ github.sha }}
      - name: Restore linux-aarch64 Libraries from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
            core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
          key: nativelibs-armlinux-${{ github.sha }}
      - name: Restore Windows CXX Library from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
          key: nativelibs-windows-${{ github.sha }}
      - name: Restore Windows Rust Library from Cache
        uses: actions/cache/restore@v3
        with:
          path: |
            core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
          key: nativelibs-windows-rust-${{ github.sha }}
      - name: Commit the files
        run: |
          git config --global user.name 'GitHub Actions - Rebuild Native Libraries'
          git config --global user.email 'jaromir@questdb.io'
          git add core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
          git add core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
          git add core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
          git add core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
          git add core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
          git add core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
          git add core/src/main/resources/io/questdb/bin/linux/libquestdb.so
          git add core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
          git add core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
          git add core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
          git add core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
          git add core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
          git commit -m "chore(build): Rebuild Native Libraries"
      - name: Check git status
        run: |
          git status
      - name: Print file sizes after
        run: |
          find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
      - name: Push the files to the current branch
        uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df
        # Why do we use a commit hash instead of a tag for the github-push-action?
        # ad-m/github-push-action is not as well-known repo as e.g. actions/checkout, and therefore we trust it less.
        # d91a48109 is the same as the tag v0.8.0, but it's guaranteed to be immutable.
        # So even if a bad actor takes over the repo, and rewrites tags to point to malicious commits, we will still be safe.
        with:
          branch: ${{ github.head_ref || github.ref_name }}