ci.yaml

name: CI

on:
  push:
    # Run on the main branch
    branches:
      - main
    # Releases are tags named 'v<version>', and must have the "major.minor.micro", for example: "0.1.0".
    # Release candidates are tagged as `v<version>-rc<num>`, for example: "0.1.0-rc1".
    tags:
      - "v*"
  # Also on PRs, just be careful not to publish anything
  pull_request:

env:
  VERSION_HELM: "v3.9.3"
  CONTAINER: docker

jobs:

  prepare:
    runs-on: ubuntu-22.04

    outputs:
      do-build: ${{steps.changed-files-irrelevant.outputs.only_modified != 'true' || steps.state.outputs.release == 'true'}}
      do-publish: ${{steps.state.outputs.do-publish}}
      release: ${{steps.state.outputs.release}}
      version: ${{steps.version.outputs.version}}
      prerelease: ${{steps.state.outputs.prerelease}}

    steps:

      - uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: 0

      # Gather information for build

      - name: Evaluate state
        id: state
        run: |
          test -z "${{github.head_ref}}" && (echo 'do-publish=true' >> $GITHUB_OUTPUT )
          if [[ "${{ github.event.ref }}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
              echo "prerelease=false" >> $GITHUB_OUTPUT
          elif [[ "${{ github.event.ref }}" =~ ^refs/tags/v.*$ ]]; then
              echo "release=true" >> $GITHUB_OUTPUT
              echo "prerelease=true" >> $GITHUB_OUTPUT
          fi

      - name: Set version
        if: ${{github.head_ref == ''}}
        id: version
        run: |
          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
          [ "$VERSION" == "main" ] && VERSION=latest
          echo VERSION=$VERSION
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "VERSION=$VERSION" >> $GITHUB_ENV

      # evaluate changed files
      - name: Evaluate non-relevant changed files
        id: changed-files-irrelevant
        uses: tj-actions/changed-files@v34
        with:
          files: |
            docs/**

  info:
    name: info
    runs-on: ubuntu-22.04
    needs: [prepare]
    steps:
      - run: |
          echo "do-build: ${{needs.prepare.outputs.do-build}}"
          echo "do-publish: ${{needs.prepare.outputs.do-publish}}"
          echo "release: ${{steps.state.outputs.release}}"
          echo "version: ${{steps.version.outputs.version}}"
          echo "prerelease: ${{steps.state.outputs.prerelease}}"

  check:
    runs-on: ubuntu-22.04
    needs: [prepare]
    if: needs.prepare.outputs.do-build == 'true'
    steps:

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            .cargo-container-home
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Check
        run: make host-check

  test:
    name: test
    runs-on: ubuntu-22.04
    needs: [prepare, check]
    if: needs.prepare.outputs.do-build == 'true'
    steps:

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            .cargo-container-home
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Run tests
        env:
          RUST_BACKTRACE: "1"
        run: |
          make test

  build:
    name: build
    runs-on: ubuntu-22.04
    needs: [prepare, check]
    if: needs.prepare.outputs.do-build == 'true'
    steps:

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            .cargo-container-home
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - uses: actions/cache@v3
        with:
          path: console-frontend/node_modules
          key: ${{ runner.os }}-node-modules-${{ hashFiles('console-frontend/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-modules-

      - name: Run build
        run: |
          make build build-images SKIP_SERVER=1

      - name: Save images
        run: |
          make save-images SKIP_SERVER=1

      - name: Upload images
        uses: actions/upload-artifact@v3
        with:
          name: container-images
          path: build/images/all.tar
          if-no-files-found: error

  server-linux:
    runs-on: ubuntu-22.04
    needs: [prepare, check]
    if: needs.prepare.outputs.do-build == 'true'
    env:
      VERSION: ${{needs.prepare.outputs.version}}
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-server-${{ hashFiles('**/Cargo.lock') }}

      - name: Install dependencies
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          sudo apt update
          sudo apt install -y build-essential curl tzdata libssl-dev pkg-config libsasl2-dev librdkafka-dev libpq-dev libpqxx-dev

      - name: Build binary
        run: |
          PQ_LIB_STATIC=1 SASL2_STATIC=1 OPENSSL_STATIC=1 cargo build --release --features static -p drogue-cloud-server

      - name: Build frontend
        run: |
          make host-frontend

      - name: Build container image
        run: |
          # we have a pre-built binary, so we can skip the main build step
          make SKIP_BUILD=1 'build-image(server)'

      - name: Export container image
        run: |
          make -C server save-images
          mv build/images/all.tar build/images/server.tar

      - name: Upload server binary
        uses: actions/upload-artifact@v3
        with:
          name: drogue-server-linux-amd64
          path: target/release/drogue-cloud-server
          if-no-files-found: error

      - name: Upload images
        uses: actions/upload-artifact@v3
        with:
          name: container-image-server
          path: build/images/server.tar
          if-no-files-found: error


  server-macos:
    runs-on: macos-12
    needs: [prepare, check]
    if: needs.prepare.outputs.do-build == 'true'
    env:
      VERSION: ${{needs.prepare.outputs.version}}
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-server-${{ hashFiles('**/Cargo.lock') }}


      - name: Fixup brew
        run: |
          # Unlink and re-link to prevent errors when github mac runner images
          # install python outside of brew, for example:
          # https://github.com/orgs/Homebrew/discussions/3895
          # https://github.com/actions/setup-python/issues/577
          # https://github.com/actions/runner-images/issues/6459
          # https://github.com/actions/runner-images/issues/6507
          # https://github.com/actions/runner-images/issues/2322
          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done

      - run: brew update

      - name: Install dependencies
        shell: bash
        run: |
          brew install librdkafka openssl

      - name: Build postgres
        shell: bash
        run: |
          # Build and install postgresql dep
          curl -fsSL -o postgresql-11.13.tar.gz https://ftp.postgresql.org/pub/source/v11.13/postgresql-11.13.tar.gz
          tar xvf postgresql-11.13.tar.gz
          cd postgresql-11.13
          ./configure --prefix=$PWD/../libpq
          make
          make install
          cd ..

      - name: Build binary
        shell: bash
        run: |
          PQ_LIB_STATIC=1 PQ_LIB_DIR=$PWD/libpq/lib SASL2_STATIC=1 OPENSSL_STATIC=1 OPENSSL_DIR=/usr/local/opt/openssl DEP_OPENSSL_DIR=/usr/local/opt/openssl RUSTFLAGS="-C target-feature=+crt-static" cargo build --release -p drogue-cloud-server --features static

      - name: Upload server binary
        uses: actions/upload-artifact@v3
        with:
          name: drogue-server-macos-amd64
          path: target/release/drogue-cloud-server
          if-no-files-found: error

          #- name: Build binary for Windows
          #  if: ${{ matrix.os == 'windows-2019' }}
          #  run: |
          #    git clone https://github.com/microsoft/vcpkg
          #    .\vcpkg\bootstrap-vcpkg.bat
          #    .\vcpkg\vcpkg install openssl
          #    .\vcpkg\vcpkg install libpq
          #    .\vcpkg\vcpkg install librdkafka
          #    .\vcpkg\vcpkg integrate install
          #    #$env:VCPKG_ROOT = "vcpkg"

          #    $env:PQ_LIB_STATIC = "1"
          #    $env:SASL2_STATIC = "1"
          #    $env:OPENSSL_STATIC = "1"
          #    $env:RUSTFLAGS = "-C target-feature=+crt-static"
          #    cargo build --release -p drogue-cloud-server --features static


  deploy:
    name: deploy on kind
    runs-on: ubuntu-22.04
    needs: [prepare, build]
    if: needs.prepare.outputs.do-build == 'true'
    env:
      CLUSTER: "kind"
    steps:

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Set up Helm
        uses: azure/setup-helm@v3
        with:
          version: ${{ env.VERSION_HELM }}

      - name: Fetch container images
        uses: actions/download-artifact@v3
        with:
          name: container-images

      - name: Create k8s kind Cluster
        uses: helm/kind-action@v1.4.0
        with:
          wait: 300s
          cluster_name: kind
          config: deploy/kind/cluster-config.yaml

      - name: Load images in docker
        run: docker load -i all.tar

      - name: Tag images
        run: |
          make tag-images CONTAINER_REGISTRY=dev.local IMAGE_TAG=${{ github.sha }} SKIP_SERVER=1 SKIP_BUILD=1 SKIP_BUILD_IMAGES=1

      - name: Load images in kind
        run: make kind-load CONTAINER_REGISTRY=dev.local IMAGE_TAG=${{ github.sha }} SKIP_SERVER=1

      - name: Delete loaded images
        run: rm all.tar

      - name: List all images in kind
        run: docker exec -t kind-control-plane crictl images

      - name: Deploy drogue
        env:
          DEBUG: "true"
        run: |
          ./scripts/drgadm deploy \
            -m \
            -e \
            -s drogueCloudCore.defaults.images.repository=dev.local \
            -S drogueCloudCore.defaults.images.tag=${{ github.sha }} \
            -s drogueCloudCore.defaults.images.pullPolicy=Never \
            -s drogueCloudExamples.defaults.images.repository=dev.local \
            -S drogueCloudExamples.defaults.images.tag=${{ github.sha }} \
            -s drogueCloudExamples.defaults.images.pullPolicy=Never
        timeout-minutes: 30

      - name: Collect logs
        if: failure()
        run: |
          mkdir -p /tmp/logs/kind
          kind export logs /tmp/logs/kind
          mkdir -p /tmp/logs/ns
          for ns in kafka kourier-system knative-serving knative-eventing drogue-iot; do
            ./.github/scripts/collect_logs.sh /tmp/logs/ns/$ns $ns
          done

      - name: Upload artifacts
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: drogue-iot-logs
          path: /tmp/logs


  publish:
    needs: [prepare, deploy, test, server-linux, server-macos]
    if: (needs.prepare.outputs.do-build == 'true') && (needs.prepare.outputs.do-publish == 'true')

    runs-on: ubuntu-22.04

    env:
      CONTAINER_REGISTRY: ghcr.io/${{ github.repository_owner }}
      VERSION: ${{needs.prepare.outputs.version}}

    steps:

      - name: Dump state
        run: |
          echo "Tag/version: ${VERSION} / ${{needs.prepare.outputs.version}}"
          echo "Should we publish artifacts? - do-publish = ${{needs.prepare.outputs.do-publish}}"
          echo "Release? - release = ${{needs.prepare.outputs.release}}"
          echo "Pre-release? - prerelease = ${{needs.prepare.outputs.prerelease}}"

      # Start the actual build

      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Set up Helm
        uses: azure/setup-helm@v3
        with:
          version: ${{ env.VERSION_HELM }}

      - name: Create installer archives
        run: |
          make -C installer VERSION=$VERSION

      - uses: actions/upload-artifact@v3
        with:
          name: installers
          path: installer/build/drogue-install-*.zip
          if-no-files-found: error

      - name: Fetch container images
        uses: actions/download-artifact@v3
        with:
          name: container-images

      - name: Fetch container image server
        uses: actions/download-artifact@v3
        with:
          name: container-image-server

      - name: Fetch static server binary (linux-amd64)
        uses: actions/download-artifact@v3
        with:
          name: drogue-server-linux-amd64
          path: drogue-server-linux-amd64

      - name: Fetch static server binary (macos-amd64)
        uses: actions/download-artifact@v3
        with:
          name: drogue-server-macos-amd64
          path: drogue-server-macos-amd64

      - name: Rename static server binaries
        run: |
          mkdir server-binaries
          mv drogue-server-linux-amd64/drogue-cloud-server server-binaries/drogue-cloud-server-linux-amd64
          mv drogue-server-macos-amd64/drogue-cloud-server server-binaries/drogue-cloud-server-macos-amd64

      - name: Load images in docker
        run: |
          docker load -i all.tar
          docker load -i server.tar
          docker images

      - name: Login to github container repository
        run: echo "${{ secrets.GHCR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Tag and push images
        run: |
          make push "IMAGE_TAG=$VERSION" SKIP_SERVER=1 SKIP_BUILD=1 SKIP_BUILD_IMAGES=1
          sleep 10 # delay a bit for API rate limiting issues

      - name: Create release notes
        run: |
          # I know it feels like overkill to use cat and redirect, but it's a pattern for future extension of the
          # release notes file. We could just add stuff.
          cat installer/README.md > /tmp/notes.md

      - name: Create Release
        if: needs.prepare.outputs.release
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG: v${{ needs.prepare.outputs.version }}
        run: |
          # we need to create and upload in a single step, otherwise the GitHub API might return "not found" on releases
          OPTS=""
          if [[ "${{ needs.prepare.outputs.prerelease }}" == "true" ]]; then
            OPTS="${OPTS} -p"
          fi
          INSTALLERS=$(find installer/build -type f -name "drogue-install-*.zip" -printf "%p ")
          SERVER_BINS=$(find server-binaries -type f -printf "%p ")
          gh release create \
            $OPTS \
            --title "${{ needs.prepare.outputs.version }}" \
            -F /tmp/notes.md \
            $TAG \
            $INSTALLERS \
            $SERVER_BINS