build-test.yaml

# Workflow triggered by pushes to the main branch and PRs targeting it
# (as long as the commits contain code changes).
#
# Jobs:
# - rustfmt: Checks formatting of the Rust sources using the 'rustfmt'
#            tool. The job is skipped if the workflow was triggered by a
#            PR marked as a draft.
# - fourmolu: Checks formatting of the Haskell sources using the
#             'fourmolu' tool. The job is skipped if the workflow was
#             triggered by a PR marked as a draft.
# - build-test: Build and test both the Haskell and Rust sources.
#               The job is skipped if the workflow was triggered by a PR
#               marked as a draft.
# 
# The steps in 'build-test' are ordered to fail as fast as possible
# and restore caches as late as possible.
# The dependencies between the steps are described in inline comments below
# along with a few suggestions for improving parallelization.

name: Check formatting, build and run tests

on:
  push:
    branches: main
    paths:
    - '.github/workflows/build-test.yaml'
    - '**/Cargo.toml'
    - '**/Cargo.lock'
    - '**.rs'
    - '**.yaml'
    - '**.hs'
    - 'concordium-base'
    - 'concordium-consensus/smart-contracts'
    - 'concordium-consensus/haskell-lmdb'
    - 'concordium-node/rustfmt.toml'

  pull_request:
    branches: main
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
    - '.github/workflows/build-test.yaml'
    - '**/Cargo.toml'
    - '**/Cargo.lock'
    - '**.rs'
    - '**.yaml'
    - '**.hs'
    - 'concordium-base'
    - 'concordium-consensus/smart-contracts'
    - 'concordium-consensus/haskell-lmdb'
    - 'concordium-node/rustfmt.toml'
  workflow_dispatch: # allow manual trigger

env:
  dummy: 16                 # change to force cache invalidation
  CARGO_TERM_COLOR: always # implicitly adds '--color=always' to all cargo commands
  TEST_LEVEL: 1            # for stack tests

jobs:

  fourmolu:
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}
    steps:

    - name: Download fourmolu
      uses: supplypike/setup-bin@v1
      with:
        uri: 'https://github.com/fourmolu/fourmolu/releases/download/v0.13.1.0/fourmolu-0.13.1.0-linux-x86_64'
        name: 'fourmolu'
        version: '0.13.1.0'

    - name: Checkout project
      uses: actions/checkout@v2
      with:
        submodules: recursive

    # Since fourmolu uses the project cabal file to find default language extensions, this step is
    # needed to generate a cabal file from the package.yaml.
    - name: Setup the project
      run: |
        stack --stack-yaml concordium-consensus/stack.yaml setup

    - name: Run fourmolu
      run: |
        fourmolu --color always --mode check $(git ls-files '*.hs')

  rustfmt:
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}

    strategy:
      matrix:
        plan:
        - rust: "nightly-2023-04-01-x86_64-unknown-linux-gnu"

    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        # token: ${{ secrets.CONCORDIUM_CI }}
        submodules: recursive

    - name: Install Rust
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: ${{ matrix.plan.rust }}
        override: true
        components: rustfmt

    - name: Run rustfmt
      run: |
        cargo fmt --manifest-path concordium-node/Cargo.toml -- --check
        cargo fmt --manifest-path collector-backend/Cargo.toml -- --check
        cargo fmt --manifest-path collector/Cargo.toml -- --check
        cargo fmt --manifest-path service/windows/Cargo.toml -- --check
        cargo fmt --manifest-path service/windows/installer/custom-actions/Cargo.toml -- --check

  build:
    needs: [fourmolu, rustfmt]
    # Use fixed OS version because we install packages on the system.
    runs-on: ubuntu-20.04
    if: ${{ !github.event.pull_request.draft }}

    strategy:
      matrix:
        plan:
        - rust: 1.73
          ghc: 9.6.4

    steps:
    - name: Remove unnecessary files
      run: |
        sudo rm -rf /usr/share/dotnet
        sudo rm -rf "$AGENT_TOOLSDIRECTORY"    
    - name: Checkout
      uses: actions/checkout@v2
      with:
        #token: ${{ secrets.CONCORDIUM_CI }}
        submodules: recursive
    - name: Install system packages and protoc
      run: |
        sudo apt-get update && sudo apt-get -y install liblmdb-dev gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64
        wget https://github.com/google/flatbuffers/archive/refs/tags/v22.12.06.zip
        unzip v22.12.06.zip
        cd flatbuffers-22.12.06
        cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release
        make
        sudo make install
        cd ..
        flatc --version
        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.3/protoc-3.15.3-linux-x86_64.zip
        unzip protoc-3.15.3-linux-x86_64.zip
        sudo mv ./bin/protoc /usr/bin/protoc

    # Set up Rust and restore dependencies and targets from cache.
    # This must be done before checking the Rust sources (obviously)
    # but also before building the Haskell sources because the Haskell
    # build kicks of a Rust build.
    - name: Install Rust
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: ${{ matrix.plan.rust }}
        override: true
        components: clippy
        target: x86_64-pc-windows-gnu
    - name: Cache cargo dependencies and targets
      uses: actions/cache@v2
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          concordium-base/rust-src/target
          concordium-base/lib
          concordium-consensus/smart-contracts/wasm-chain-integration/target
          concordium-consensus/smart-contracts/lib
          concordium-node/target
        key: ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}

    # HASKELL #

    # Set up Haskell by caching '~/.stack', '.stack-work', and '~/.local/bin' separately.
    # This must be done before compiling the Haskell sources
    # (which in turns compiles certain Rust sources).
    # The cache entry keys depend on the GHC version and contents of 'package.yaml' and 'stack.yaml'
    # but will fall back to cache entries from different versions if no match is found.

    - name: Cache stack global package DB
      id: stack-global
      uses: actions/cache@v2
      with:
        path: ~/.stack
        key: ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}
    - name: Cache stack-installed programs in '~/.local/bin'
      id: stack-programs
      uses: actions/cache@v2
      with:
        path: ~/.local/bin
        key: ${{ runner.os }}-${{ env.dummy }}-stack-programs-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-stack-programs-${{ matrix.plan.ghc }}
    - name: Cache '.stack-work'
      uses: actions/cache@v2
      with:
        path: |
          .stack-work
          concordium-base/.stack-work
          concordium-consensus/.stack-work
          concordium-consensus/haskell-lmdb/.stack-work

        key: ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}

    - name: Install GHC (unless it was cached)
      if: steps.stack-programs.outputs.cache-hit != 'true' || steps.stack-global.outputs.cache-hit != 'true'
      run: |
        stack setup --install-ghc
    - name: Build haskell dependencies (unless they were cached)
      if: steps.stack-programs.outputs.cache-hit != 'true' || steps.stack-global.outputs.cache-hit != 'true'
      run: |
        stack build --test --bench --only-dependencies --stack-yaml concordium-consensus/stack.yaml

    # Compile Haskell sources. This must be done before running checks or tests on the Rust sources.
    - name: Build consensus
      run: |
        stack build --test --bench --force-dirty --stack-yaml concordium-consensus/stack.yaml --no-run-tests --no-run-benchmarks --ghc-options "-Werror"

    # Test Haskell sources. Could be run in parallel with the steps below.
    - name: Test consensus
      run: |
        stack --stack-yaml concordium-consensus/stack.yaml test concordium-consensus:consensus --bench --no-run-benchmarks --ta --level=${TEST_LEVEL}
    - name: Test globalstate
      run: |
        stack --stack-yaml concordium-consensus/stack.yaml test concordium-consensus:globalstate --bench --no-run-benchmarks --ta --level=${TEST_LEVEL}
    - name: Test scheduler
      run: |
        stack --stack-yaml concordium-consensus/stack.yaml test concordium-consensus:scheduler --bench --no-run-benchmarks

    # RUST #

    # Check, compile, and test Rust sources. All the steps below could be run in parallel in separate jobs.
    - name: Check that Rust target compiles
      run: |
        cargo check --locked --manifest-path concordium-node/Cargo.toml --workspace
    - name: Check that the collector-backend compiles
      run: |
        cargo check --locked --manifest-path collector-backend/Cargo.toml --workspace
    - name: Check that the collector compiles
      run: |
        cargo check --locked --manifest-path collector/Cargo.toml
    - name: Check that the Windows node runner service compiles (gnu toolchain, disable embedded resources)
      run: |
        cargo check --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/Cargo.toml --workspace --no-default-features
    - name: Check that the Windows installer custom action library compiles (gnu toolchain)
      run: |
        cargo check --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/installer/custom-actions/Cargo.toml --workspace
    - name: Run clippy (without extra features)
      run: |
        cargo clippy --locked --manifest-path concordium-node/Cargo.toml --all -- -D warnings
    - name: Run clippy (with feature 'network_dump')
      run: |
        cargo clippy --locked --manifest-path concordium-node/Cargo.toml --features=network_dump --all -- -D warnings
    - name: Run clippy on collector backend
      run: |
        cargo clippy --locked --manifest-path collector-backend/Cargo.toml -- -D warnings
    - name: Run clippy on collector
      run: |
        cargo clippy --locked --manifest-path collector/Cargo.toml -- -D warnings
    - name: Run clippy on Windows node runner service
      run: |
        cargo clippy --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/Cargo.toml --all --no-default-features -- -D warnings
    - name: Run clippy on Windows installer custom action library
      run: |
        cargo clippy --locked --target x86_64-pc-windows-gnu --manifest-path service/windows/installer/custom-actions/Cargo.toml --all -- -D warnings
    - name: Test Rust crates (without extra features)
      run: |
        cargo test --manifest-path concordium-node/Cargo.toml --all
    - name: Test Rust crates (with feature 'network_dump')
      run: |
        cargo test --manifest-path concordium-node/Cargo.toml --all --features=network_dump