CI with Github Actions (#1)

Move from BB pipelines to Github actions, add documentation.

.github/workflows/deploy.yml

@@ -0,0 +1,33 @@
+name: deploy-to-staging
+
+on: 
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    name: "Deploy (main to staging)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for tests to pass
+        uses: lewagon/wait-on-check-action@v0.2
+        with:
+          ref: ${{ github.ref }}
+          # check-name: "Test (on Python3.8)" # name of the job we wait for (omit to wait for all checks)
+          running-workflow-name: "Deploy (main to staging)"  # name of the check that will wait for other checks
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 20 # seconds
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+          ref: 'main'
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_DEPLOYMENT_KEY }}  # private ssh key
+          known_hosts: ${{ secrets.KNOWN_DEPLOYMENT_HOSTS }}  # make via ssh-keyscan -t rsa <your host>
+      - run: ci/DEPLOY.sh
+    env:
+      BRANCH_NAME: main
+      STAGING_REMOTE_REPO: ${{ secrets.STAGING_REMOTE_REPO }}

.github/workflows/lint-and-test.yml

@@ -0,0 +1,50 @@
+name: lint-and-test
+
+on: push
+
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    name: Check (on Python3.8)
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/checkout@v2
+      - uses: pre-commit/action@v2.0.0
+
+  test:
+    needs: check
+    runs-on: ubuntu-latest
+    #strategy:  # TODO: make work with pre-commit
+    #  matrix:
+    #    python-version: [ '3.8', '3.6' ]
+    name: "Test (on Python3.8)"
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/checkout@v2
+      - run: ci/SETUP.sh
+      - run: make test
+    env:
+      PGHOST: 127.0.0.1
+      PGPORT: 5432
+      PGUSER: flexmeasures_test
+      PGDB: flexmeasures_test
+      PGPASSWORD: flexmeasures_test
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:12.5 
+        env:
+          POSTGRES_USER: flexmeasures_test
+          POSTGRES_PASSWORD: flexmeasures_test
+          POSTGRES_DB: flexmeasures_test
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

Makefile

@@ -15,41 +15,41 @@ test:
 # ---- Documentation ---
 
 update-docs:
-	pip install sphinx sphinxcontrib.httpdomain
+	pip3 install sphinx sphinxcontrib.httpdomain
 	cd documentation; make clean; make html; cd ..
 
 update-docs-pdf:
 	@echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)"
 	@echo "NOTE: Currently, the docs require some pictures which are not in the git repo atm. Ask the devs."
-	pip install sphinx sphinxcontrib.httpdomain
+	pip3 install sphinx sphinxcontrib.httpdomain
 	cd documentation; make clean; make latexpdf; make latexpdf; cd ..  # make latexpdf can require two passes
 
 # ---- Installation ---
 
 install: install-deps install-flexmeasures
 
 install-for-dev:
-	pip install -q pip-tools
+	pip3 install -q pip-tools
 	make freeze-deps
 	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
 	make install-flexmeasures
 
 install-deps:
-	pip install -q pip-tools
+	pip3 install -q pip-tools
 	make freeze-deps
 	pip-sync requirements/app.txt
 
 install-flexmeasures:
 	python setup.py develop
 
 freeze-deps:
-	pip install -q pip-tools
+	pip3 install -q pip-tools
 	pip-compile -o requirements/app.txt requirements/app.in
 	pip-compile -o requirements/dev.txt requirements/dev.in
 	pip-compile -o requirements/test.txt requirements/test.in
 
 upgrade-deps:
-	pip install -q pip-tools
+	pip3 install -q pip-tools
 	pip-compile --upgrade -o requirements/app.txt requirements/app.in
 	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in

Readme.md

@@ -1,7 +1,14 @@
 # The FlexMeasures Platform
 
+![lint-and-test](https://github.com/SeitaBV/flexmeasures/workflows/lint-and-test/badge.svg)
+[![](https://img.shields.io/badge/python-3.6+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
 The *FlexMeasures Platform* is a tool for scheduling flexible actions for energy assets.
-For this purpose, it performs monitoring, forecastig and scheduling services.
+For this purpose, it performs monitoring, forecasting and scheduling services.
+
+Its role is to enhance energy services. Forecasts and schedules are made available via API.
+
 
 ## Build & Run
 

ci/DEPLOY.sh

@@ -7,4 +7,4 @@
 git remote add staging $STAGING_REMOTE_REPO
 
 # Push the branch being deployed to the git remote. Also push any annotated tags (with a -m message).
-git push --follow-tags --set-upstream staging $BITBUCKET_BRANCH
+git push --follow-tags --set-upstream staging $BRANCH_NAME

ci/README.md

@@ -2,7 +2,7 @@
 
 Here you can learn how to get FlexMeasures onto a server.
 
-We talk about serving FlexMeasures in a WSGI setting and deploying on a server via git.
+We talk about serving FlexMeasures in a WSGI setting, installing the linear solver and deploying on a (staging) server via git.
 
 TODO: Dockerization
 
@@ -14,8 +14,7 @@ Here is an example how to serve this application as WSGI app:
 
     # This file contains the WSGI configuration required to serve up your
     # web application.
-    # It works by setting the variable 'application' to a WSGI handler of some
-    # description.
+    # It works by setting the variable 'application' to a WSGI handler of some description.
 
     import sys
     import os
@@ -35,47 +34,71 @@ Here is an example how to serve this application as WSGI app:
 
 ## Install the linear solver on the server
 
-To compute schedules, we use a linear optimization solver.
-The Cbc solver has to be installed from source, on the server where FlexMeasures runs.
+To compute schedules, FlexMeasures uses the [Cbc](https://github.com/coin-or/Cbc) mixed integer linear optimization solver.
+It is used through [Pyomo](http://www.pyomo.org), so in principle supporting a [different solver](https://pyomo.readthedocs.io/en/stable/solving_pyomo_models.html#supported-solvers) would be possible.
+
+Cbc needs to be present on the server where FlexMeasures runs, under the `cbc` command.
+
+You can install it on Debian like this:
+
+    apt-get install coinor-cbc
+
+If you can't use the package manager on your host, the solver has to be installed from source.
 We provide [an example script](ci/install-cbc.sh) to do that, where you can also
 pass a directory for the installation.
+
 In case you want to install a later version, adapt the version in the script. 
 
 
-## Automate deployment via Bitbucket Pipelines
+## Automate deployment via Github actions
+
+Github action workflows are in the `.github/workflows` directory. We use them to build and deploy the project to our staging server.
+
+Documenting this might be useful for self-hosters, as well.
+The GitHub Actions workflows are triggered by commits being pushed to the repository, but it can also inspire your custom deployment script.
 
-The Bitbucket Pipeline is configured with the `bitbucket-pipelines.yml` file.
-It reacts to commits being made to the repository, but it can also inspire your custom deployment script.
-In this file we set up the app, run the tests and linters, and if the commit was on the master branch,
-we finish with deploying the code to a staging server (see below).
+We'll refer to Github Actions as our "CI environment" and our staging server as the "deployment server". 
 
+- In `lint-and-test.yml`, we set up the app, then run the tests and linters.
+If testing succeeds and if the commit was on the `main` branch, `deploy.yml` deploys the code from the CI environment to the deployment server.
 
-## Deployment on the server via Git
+- Of course, the CI environment needs to properly authenticate at the deployment server. 
+
+- With the hooks functionality of Git, a post-receive script can then (re-)start the FlexMeasures app on the deployment server.
+
+Let's review these three steps in detail:
+
+### Using git to deploy code (remote upstream)
 
 We support deployment of the FlexMeasures project on a staging server via Git checkout.
 
-The deployment uses git's ability to push code to a remote upstream repository.
-We trigger this deployment in `bitbucket-pipelines.yml` (see above)
-With the hooks functionality of Git, a post-receive script can then (re)start the FlexMeasures app.
+The deployment uses git's ability to push code to a remote upstream repository. This repository needs to be installed on your staging server.
+
+We trigger this deployment in `deploy.yml` and it's being done in `DEPLOY.sh`. There, we add the remote and then push the current branch to it.
+
+We thus need to tell the deployment evnironment two things:
+
+- Add the setting `STAGING_REMOTE_REPO` as an environment variable on the deployment environment (e.g. `deploy.yml` expects it in the Github repository secrets). An example value is `seita@ssh.our-server.com:/home/seita/flexmeasures-staging/flexmeasures.git`.
+- Make sure the env variable `BRANCH_NAME` is set, e.g. to "main", so that the deployment environment knows what exact code to push to your deployment server.
 
-### Remote origin
+### Authenticate at the deployment server (with an ssh key)
 
-To see how a remote repo is added, see `DEPLOY.sh`. There, we add the remote and also push the current branch there.
+The CI environment needs to authenticate at the deployment server using an SSH key pair (use `ssh-keygen` to create one, using no password).
 
-To make this work, we need three things:
+To make this work, we need to configure the following:
 
-- Make sure the remote git repo exists (is cloned)
-- Add the setting `STAGING_REMOTE_REPO` to the deployment environment (e.g. Bitbucket pipelines). An example value is `seita@ssh.our-server.com:/home/seita/flexmeasures-staging/flexmeasures.git`.
-- Set up an SSH key for deployment in the deployment environment, so that the server accepts the code.
+- Add the deployment server to `~/.ssh/known_hosts` of the deployment environment, so that the deployment environment knows it's okay to talk to the deployment server (e.g. `deploy.yml` expects it in the Github repository secrets as `KNOWN_DEPLOYMENT_HOSTS`). You can create this entry with `ssh-keyscan -t rsa <your host>`.
+- Add the private part of the ssh key pair as key in the deployment environment, so that the deployment server can accept the pushed code. (e.g. as `~/.ssh/id_rsa`). In `deploy.yml`, we expect it as the secret `SSH_DEPLOYMENT_KEY`, which addds the key for us.
+- Finally, the public part of the key pair should be in `~/.ssh/authorized_keys` on your deployment server.
 
+### (Re-)start FlexMeasures on the deployment server (install Post-Receive Hook)
 
-### Install Post-Receive Hook
+Only pushing the code will not actually deploy the updated FlexMeasures into a usable web app on the deployment server. For this, we need to trigger a script.
 
-Only pushing the code will not deploy the updated FlexMeasures. For this, we need to trigger a script.
-Log on to the server (via SSH) and install the Git Post Receive Hook in the remote repo where we deployed the code (see above). This hook will be triggered when a push is received from the deployment environment.
+Log on to the server (via SSH) and install a script to (re-)start FlexMeasures as a Git Post Receive Hook in the remote repo where we deployed the code (see above). This hook will be triggered when a push is received from the deployment environment.
 
 The example script below can be a Post Receive Hook (save as `hooks/post-receive` in your remote origin repo and update paths).
-It will force checkout the master branch, update dependencies, upgrade the database structure,
+It will force checkout the main branch, update dependencies, upgrade the database structure,
 update the documentation and finally touch the wsgi.py file.
 This last step is often a way to soft restart the running application, but here you need to adapt to your circumstances.
 

ci/SETUP.sh

@@ -12,9 +12,7 @@ head -c 24 /dev/urandom > ./instance/secret_key
 
 # Install dependencies
 apt-get update
-apt-get -y install postgresql-client coinor-cbc
-# set PGDB, PGUSER and PGPASSWORD as envs for this
-psql -h localhost -p 5432 -c "create extension if not exists cube; create extension if not exists earthdistance;" -U $PGUSER $PGBD;
+sudo apt-get -y install postgresql-client coinor-cbc
 make install-deps
 
 
@@ -23,7 +21,7 @@ make install-deps
 # Hack until this feature is ready: https://bitbucket.org/site/master/issues/15244/build-execution-should-wait-until-all
 statusFile=/tmp/postgres-status
 while [[ true ]]; do
-  telnet 127.0.0.1 5432 &> ${statusFile}
+  telnet $PGHOST $PGPORT &> ${statusFile}
   status=$(grep "Connection refused" ${statusFile} | wc -l)
   echo "Status: $status"
 
@@ -36,3 +34,5 @@ while [[ true ]]; do
     break;
   fi
 done
+
+psql -h $PGHOST -p $PGPORT -c "create extension if not exists cube; create extension if not exists earthdistance;" -U $PGUSER $PGDB;

flexmeasures/utils/config_defaults.py

@@ -129,7 +129,7 @@ class TestingConfig(Config):
 
     SECURITY_PASSWORD_SALT = "$2b$19$abcdefghijklmnopqrstuv"
     SQLALCHEMY_DATABASE_URI = (
-        "postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1/flexmeasures_test"
+        "postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test"
     )
     # SQLALCHEMY_ECHO = True
     FLEXMEASURES_TASK_CHECK_AUTH_TOKEN = "test-task-check-token"