Merge pull request #2149 from jorisv/topic/accelerate_ci

Accelerate CI
stack-of-tasks · Feb 2, 2024 · 6b5bc24 · 6b5bc24
2 parents d7c4b8c + 0fab578
commit 6b5bc24
Show file tree

Hide file tree

Showing 5 changed files with 153 additions and 31 deletions.
diff --git a/.github/workflows/check-changelog.yml b/.github/workflows/check-changelog.yml
@@ -1,4 +1,4 @@
-name: Check-changelog
+name: CI - Check-changelog
 on:
   pull_request:
     types: [assigned, opened, synchronize, reopened, labeled, unlabeled]

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -1,6 +1,29 @@
 name: CI - Linux via APT
 
-on: [push,pull_request]
+on:
+  push:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
@@ -12,7 +35,10 @@ jobs:
         os: [ubuntu-22.04,ubuntu-20.04]
 
     env:
-      CCACHE_DIR: /github/home/.ccache
+      CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
+      CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
+      CCACHE_COMPRESS: true
+      CCACHE_COMPRESSLEVEL: 5
 
     steps:
     - uses: actions/checkout@v4
@@ -23,9 +49,9 @@ jobs:
     - name: Setup ccache
       uses: actions/cache@v4
       with:
-        key: ${{ matrix.os }}
-        path: ${{ env.CCACHE_DIR }}
-        max-size: 5G
+        path: .ccache
+        key: ccache-linux-${{ matrix.os }}-${{ github.sha }}
+        restore-keys: ccache-linux-${{ matrix.os }}-
 
     # extract branch name
     - name: Get branch name (merge)
@@ -51,13 +77,26 @@ jobs:
       run: |
         sudo rm -rf /usr/local/share/boost/1.69.0
         export PYTHON3_VERSION=$(python3 -c "import sys; print(str(sys.version_info.major)+str(sys.version_info.minor))")
-        export APT_DEPENDENCIES="doxygen libboost-system-dev libboost-test-dev libboost-filesystem-dev libboost-program-options-dev libeigen3-dev liburdfdom-dev texlive-font-utils"
-        export APT_DEPENDENCIES=$APT_DEPENDENCIES" libboost-python-dev robotpkg-py"$PYTHON3_VERSION"-eigenpy python3-numpy"
-        export APT_DEPENDENCIES=$APT_DEPENDENCIES" robotpkg-py"$PYTHON3_VERSION"-hpp-fcl"
+        export APT_DEPENDENCIES="doxygen \
+                                 ccache \
+                                 curl \
+                                 cppcheck \
+                                 libomp-dev \
+                                 libomp5 \
+                                 libboost-system-dev \
+                                 libboost-test-dev \
+                                 libboost-filesystem-dev \
+                                 libboost-program-options-dev \
+                                 libeigen3-dev \
+                                 liburdfdom-dev \
+                                 texlive-font-utils \
+                                 libboost-python-dev \
+                                 robotpkg-py${PYTHON3_VERSION}-eigenpy \
+                                 python3-numpy \
+                                 robotpkg-py${PYTHON3_VERSION}-hpp-fcl"
         echo $APT_DEPENDENCIES
         sudo apt-get update -qq
-        sudo apt-get install -qq curl cppcheck ${APT_DEPENDENCIES}
-        sudo apt install libomp-dev libomp5
+        sudo apt-get install -qq ${APT_DEPENDENCIES}
     - name: Free disk space
       run: |
         sudo apt clean
@@ -68,8 +107,12 @@ jobs:
         git clone https://github.com/casadi/casadi.git -b 3.5.0 --depth 1
         cd casadi
         mkdir build && cd build
-        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
-        cmake .. -DCMAKE_BUILD_TYPE=Release -DWITH_EXAMPLES=OFF
+        cmake .. \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DWITH_EXAMPLES=OFF
+        make -j2
         sudo make install
         cd ../..
         rm -rf casadi
@@ -81,14 +124,20 @@ jobs:
         export PYTHON3_DOT_VERSION=$(python3 -c "import sys; print(str(sys.version_info.major)+'.'+str(sys.version_info.minor))")
         export PYTHONPATH=${PYTHONPATH}:/opt/openrobots/lib/python$PYTHON3_DOT_VERSION/site-packages
         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/openrobots/lib:/usr/local/lib:/usr/lib:/usr/lib/x86_64-linux-gnu
-        export MAKEFLAGS="-j1"
-        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
 
         mkdir build
         cd build
-        cmake .. -DCMAKE_BUILD_TYPE=Debug -DBUILD_WITH_COLLISION_SUPPORT=ON -DBUILD_ADVANCED_TESTING=ON -DBUILD_WITH_CASADI_SUPPORT=ON -DPYTHON_EXECUTABLE=$(which python3) -DBUILD_WITH_OPENMP_SUPPORT=ON -DINSTALL_DOCUMENTATION=ON
-        make
-        make build_tests
+        cmake .. \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_BUILD_TYPE=Debug \
+          -DBUILD_WITH_COLLISION_SUPPORT=ON \
+          -DBUILD_ADVANCED_TESTING=ON \
+          -DBUILD_WITH_CASADI_SUPPORT=ON \
+          -DPYTHON_EXECUTABLE=$(which python3) \
+          -DBUILD_WITH_OPENMP_SUPPORT=ON \
+          -DINSTALL_DOCUMENTATION=ON
+        make -j2
+        make -j2 build_tests
         export CTEST_OUTPUT_ON_FAILURE=1
         make test
         sudo make install
@@ -102,15 +151,19 @@ jobs:
         mkdir build
         cd build
         export CMAKE_PREFIX_PATH=/usr/local
-        cmake -DPYTHON_EXECUTABLE=$(which python3) ..
-        make
+        cmake .. \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DPYTHON_EXECUTABLE=$(which python3)
+        make -j2
         ./run_rnea
         ./load_urdf
         cd ../../pkgconfig
         mkdir build
         cd build
-        cmake -DPYTHON_EXECUTABLE=$(which python3) ..
-        make
+        cmake .. \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DPYTHON_EXECUTABLE=$(which python3)
+        make -j2
         ./run_rnea
         cd ../../external
         export PINOCCHIO_GIT_REPOSITORY="file://"$GITHUB_WORKSPACE
@@ -119,15 +172,19 @@ jobs:
         git tag $PINOCCHIO_GIT_TAG
         mkdir build
         cd build
-        cmake -DPYTHON_EXECUTABLE=$(which python3) ..
-        make
+        cmake .. \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DPYTHON_EXECUTABLE=$(which python3)
+        make -j2
         ./run_rnea
         ./load_urdf
         cd ../../pinocchio_header
         mkdir build
         cd build
-        cmake -DPYTHON_EXECUTABLE=$(which python3) ..
-        make
+        cmake .. \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DPYTHON_EXECUTABLE=$(which python3)
+        make -j2
         ./run_fk
 
 

diff --git a/.github/workflows/macos-linux-conda.yml b/.github/workflows/macos-linux-conda.yml
@@ -1,6 +1,29 @@
 name: CI - OSX/Linux via Conda
 
-on: [push,pull_request]
+on:
+  push:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   pinocchio-conda:
@@ -10,7 +33,7 @@ jobs:
       CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
       CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
       CCACHE_COMPRESS: true
-      CCACHE_COMPRESSLEVEL: 6
+      CCACHE_COMPRESSLEVEL: 5
       BUILD_ADVANCED_TESTING: ${{ matrix.BUILD_ADVANCED_TESTING }}
 
     strategy:

diff --git a/.github/workflows/ros-ci.yml b/.github/workflows/ros-ci.yml
@@ -5,7 +5,28 @@
 name: CI - Linux via ROS
 
 # This determines when this workflow is run
-on: [push, pull_request] # on all pushes and PRs
+on:
+  push:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - '.pre-commit-config.yaml'
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - '.pre-commit-config.yaml'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   CI:

diff --git a/.github/workflows/windows-conda.yml b/.github/workflows/windows-conda.yml
@@ -1,7 +1,28 @@
 name: CI - Windows via Conda
 on:
-  pull_request:
   push:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '.gitlab-ci.yml'
+      - '.gitignore'
+      - '*.md'
+      - 'CITATION.*'
+      - 'COPYING.LESSER'
+      - 'colcon.pkg'
+      - '.pre-commit-config.yaml'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
@@ -11,7 +32,7 @@ jobs:
       CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
       CCACHE_DIR: "${GITHUB_WORKSPACE}/.ccache"
       CCACHE_COMPRESS: true
-      CCACHE_COMPRESSLEVEL: 6
+      CCACHE_COMPRESSLEVEL: 5
 
     strategy:
       fail-fast: false