forked from questdb/questdb
-
Notifications
You must be signed in to change notification settings - Fork 0
339 lines (336 loc) · 14.6 KB
/
rebuild_native_libs.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
name: Rebuild Native C++ Libraries and Push to PR
on:
workflow_dispatch:
# This workflow is triggered manually from the Actions tab.
# It's meant to be run on a PR that changes the native C++ code.
# It builds native libraries for all supported platforms and pushes them to the current branch.
# It splits the building process into 3 build jobs:
# 1. build-freebsd - Builds native libraries for x64 FreeBSD
# 2. build-macos - Builds native libraries for MacOS: both ARM and x64.
# It uses native runners for each platform, because cross compilation on MacOS is complicated.
# 3. build-others - Builds native libraries for x64 Linux, ARM Linux and Windows.
# It uses cross-compilation for ARM Linux and Windows.
#
# Each build job saves the resulting binaries to the cache under a unique key
# When all build jobs are finished, the collect-commit-and-push job restores the binaries from the cache
# and pushes them to the current branch.
jobs:
build-rust-windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
- name: Build Rust Library
run: |
cd core/rust/qdbr
cargo build --release
- name: log git status
run: |
git status
- name: Copy Rust Library to the final directory
run: |
cp core/rust/qdbr/target/release/questdbr.dll core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
- name: Save Windows Rust Library to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
key: nativelibs-windows-rust-${{ github.sha }}
build-all-freebsd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build in FreeBSD
# @d44bf830e7d is the same as the tag v1, but it's guaranteed to be immutable.
uses: vmactions/freebsd-vm@d44bf830e7d850111d27aa120d468c905e5980f4
with:
# FreeBSD 13.2
release: 13.2
usesh: true
prepare: |
pkg install -y curl cmake gcc git nasm openjdk17 rust
run: |
export JAVA_HOME=/usr/local/openjdk17
cd core
cmake -B build/release -DCMAKE_BUILD_TYPE=Release
cmake --build build/release --config Release
cd rust/qdbr
cargo build --release
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/freebsd/
- name: log git status
run: |
git status
- name: Save FreeBSD Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
key: nativelibs-freebsd-${{ github.sha }}
build-all-macos:
strategy:
matrix:
# macos-14 = ARM M1
# macos-latest = x64
# if you change OS definitions then you need to change conditions in cache-save steps below
os: [ macos-14, macos-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Install toolchains (CXX/NASM)
run: |
brew install make cmake gcc nasm
- name: Build CXX Library
run: |
cd core
cmake -B build/release -DCMAKE_BUILD_TYPE=Release
cmake --build build/release --config Release
- name: Build Rust Library
run: |
cd core/rust/qdbr
cargo clean
cargo build --release
- name: Copy darwin-aarch64 Rust Library to the final directory
if: ${{ matrix.os == 'macos-14' }}
run: |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/armosx/
- name: Copy darwin-amd64 Rust Library to the final directory
if: ${{ matrix.os == 'macos-latest' }}
run: |
cp core/rust/qdbr/target/release/libquestdbr.dylib core/src/main/resources/io/questdb/bin/osx/
- name: Save darwin-aarch64 Libraries to Cache
if: ${{ matrix.os == 'macos-14' }}
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
key: nativelibs-armosx-${{ github.sha }}
- name: Save darwin-amd64 Libraries to Cache
if: ${{ matrix.os == 'macos-latest' }}
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
key: nativelibs-osx-${{ github.sha }}
build-all-linux-amd64:
runs-on: ubuntu-latest
# this is very bare container, we use it to restrict GLIBC to 2.28
# everything has to be installed on it manually
container:
image: debian:buster
steps:
- uses: actions/checkout@v4
- name: Install toolchains (CXX/NASM/Rust)
run: |
ldd --version
env
apt-get update -y
apt install git nasm python3 wget ca-certificates build-essential zstd -y
python3 ./.github/prepare_rust_env.py
echo "PATH=/github/home/.cargo/bin/:$PATH" >> "$GITHUB_ENV"
- name: Install up-to-date CMake
run: |
wget -nv https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz
tar -zxf cmake-3.29.2-linux-x86_64.tar.gz
echo "PATH=`pwd`/cmake-3.29.2-linux-x86_64/bin/:$PATH" >> "$GITHUB_ENV"
- name: Install JDK17 (for jni.h)
run: |
wget -nv https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
tar xfz OpenJDK17U-jdk_x64_linux_hotspot_17.0.11_9.tar.gz
echo "JAVA_HOME=`pwd`/jdk-17.0.11+9/" >> "$GITHUB_ENV"
- name: Generate Makefiles
run: |
cd ./core
cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: Build linux-amd64 CXX Library
run: |
cd core
cmake --build cmake-build-release --config Release
- name: Build linux-amd64 Rust Library
run: |
cd core/rust/qdbr
cargo clean
cargo build --release
cp target/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/linux/
- name: Save linux-amd64 Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
key: nativelibs-linux-${{ github.sha }}
build-all-linux-aarch64:
runs-on: ubuntu-latest
env:
generator: "CodeBlocks - Unix Makefiles"
steps:
- uses: actions/checkout@v4
- name: Increase file count and install tooling
run: |
env
sudo sysctl -w fs.file-max=500000
sudo apt-get update -y
sudo apt-get install -y nasm g++-10-aarch64-linux-gnu gcc-10-aarch64-linux-gnu
- name: Install rs-cross
run: |
cargo install cross
- name: Generate Makefiles
run: |
cd ./core
cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: CMAKE linux-aarch64
run: |
cd ./core
cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake -DARM_TOOLCHAIN_PATH=/usr/bin/ -DCMAKE_EXECUTABLE_SUFFIX=-10 -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-arm64
- name: Build linux-aarch64 CXX Library
run: |
cd core
cmake --build cmake-build-release-arm64 --config Release
- name: Build linux-aarch64 Rust Library
run: |
cd core/rust/qdbr
cargo clean
cross build --target aarch64-unknown-linux-gnu --release --target-dir `pwd`/target
cp target/aarch64-unknown-linux-gnu/release/libquestdbr.so ../../src/main/resources/io/questdb/bin/armlinux/
- name: Check git status
run: |
git status
- name: Save linux-aarch64 Libraries to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
key: nativelibs-armlinux-${{ github.sha }}
build-cxx-windows:
runs-on: ubuntu-latest
env:
generator: "CodeBlocks - Unix Makefiles"
steps:
- uses: actions/checkout@v4
- name: Increase file count and install tooling
run: |
env
sudo sysctl -w fs.file-max=500000
sudo apt-get update -y
sudo apt-get install -y nasm gcc-mingw-w64 g++-mingw-w64
- name: Generate Makefiles
run: |
cd ./core
cmake -G "$generator" -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S.
- name: Download windows jni_md.h from JDK 11
run: |
cd core
curl https://raw.githubusercontent.com/AdoptOpenJDK/openjdk-jdk11/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h
- name: CMake Windows
run: |
cd core
cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/windows-x86_64.cmake -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-win64
- name: Build Windows C/C++ Library
run: |
cd core
cmake --build cmake-build-release-win64 --config Release
- name: Check C++ runtime dependency
run: |
cd ./core
if x86_64-w64-mingw32-objdump -p ./src/main/resources/io/questdb/bin/windows/libquestdb.dll | grep -q libstdc++; then
echo "Failure: C++ runtime dependency detected"
exit 1
fi
- name: Check git status
run: |
git status
- name: Save Windows C/C++ Library to Cache
uses: actions/cache/save@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
key: nativelibs-windows-${{ github.sha }}
collect-commit-and-push:
needs: [ build-all-freebsd, build-all-macos,build-cxx-windows, build-rust-windows, build-all-linux-amd64, build-all-linux-aarch64 ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Print file sizes before
run: |
pwd
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
- name: Restore FreeBSD Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
key: nativelibs-freebsd-${{ github.sha }}
- name: Restore darwin-aarch64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
key: nativelibs-armosx-${{ github.sha }}
- name: Restore darwin-amd64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
key: nativelibs-osx-${{ github.sha }}
- name: Restore linux-amd64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/linux/libquestdb.so
core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
key: nativelibs-linux-${{ github.sha }}
- name: Restore linux-aarch64 Libraries from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
key: nativelibs-armlinux-${{ github.sha }}
- name: Restore Windows CXX Library from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
key: nativelibs-windows-${{ github.sha }}
- name: Restore Windows Rust Library from Cache
uses: actions/cache/restore@v3
with:
path: |
core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
key: nativelibs-windows-rust-${{ github.sha }}
- name: Commit the files
run: |
git config --global user.name 'GitHub Actions - Rebuild Native Libraries'
git config --global user.email 'jaromir@questdb.io'
git add core/src/main/resources/io/questdb/bin/armosx/libquestdb.dylib
git add core/src/main/resources/io/questdb/bin/armosx/libquestdbr.dylib
git add core/src/main/resources/io/questdb/bin/osx/libquestdb.dylib
git add core/src/main/resources/io/questdb/bin/osx/libquestdbr.dylib
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdb.so
git add core/src/main/resources/io/questdb/bin/freebsd/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/linux/libquestdb.so
git add core/src/main/resources/io/questdb/bin/linux/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdb.so
git add core/src/main/resources/io/questdb/bin/armlinux/libquestdbr.so
git add core/src/main/resources/io/questdb/bin/windows/libquestdb.dll
git add core/src/main/resources/io/questdb/bin/windows/libquestdbr.dll
git commit -m "chore(build): Rebuild Native Libraries"
- name: Check git status
run: |
git status
- name: Print file sizes after
run: |
find ./core/src/main/resources/io/questdb/bin/ -type f -exec ls -l {} \;
- name: Push the files to the current branch
uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df
# Why do we use a commit hash instead of a tag for the github-push-action?
# ad-m/github-push-action is not as well-known repo as e.g. actions/checkout, and therefore we trust it less.
# d91a48109 is the same as the tag v0.8.0, but it's guaranteed to be immutable.
# So even if a bad actor takes over the repo, and rewrites tags to point to malicious commits, we will still be safe.
with:
branch: ${{ github.head_ref || github.ref_name }}