FlexMeasures · nhoening · Apr 25, 2022 · Apr 10, 2022 · Apr 11, 2022 · Apr 11, 2022
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,42 @@
+FROM ubuntu:focal
+
+# TODO: Cbc solver
+# TODO: run gunicorn as entry command
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
+
+# pre-requisites
+RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl gunicorn coinor-cbc
+
+WORKDIR /app
+# requirements - doing this earlier, so we don't install them each time. Use --no-cache to refresh them.
+COPY requirements /app/requirements
+
+# py dev tooling
+RUN python3 -m pip install --upgrade pip && python3 --version
+RUN pip3 install --upgrade setuptools
+RUN pip3 install -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt
+
+# Copy code and meta/config data
+COPY setup.* .flaskenv wsgi.py /app/
+COPY flexmeasures/ /app/flexmeasures
+RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+COPY .git/ /app/.git
+
+RUN pip3 install .
+
+EXPOSE 5000
+
+CMD [ \
+    "gunicorn", \
+    "--bind", "0.0.0.0:5000", \
+    # This is set to /tmp by default, but this is part of the Docker overlay filesystem, and can cause stalls.
+    # http://docs.gunicorn.org/en/latest/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod
+    "--worker-tmp-dir", "/dev/shm", \
+    # Ideally you'd want one worker per container, but we don't want to risk the health check timing out because
+    # another request is taking a long time to complete.
+    "--workers", "2", "--threads", "4", \
+    "wsgi:application" \
+]
diff --git a/ci/load-psql-extensions.sql b/ci/load-psql-extensions.sql
@@ -0,0 +1,2 @@
+CREATE EXTENSION IF NOT EXISTS cube;
+CREATE EXTENSION IF NOT EXISTS earthdistance;
diff --git a/ci/setup-postgres.sh b/ci/setup-postgres.sh
@@ -27,4 +27,4 @@ while [[ true ]]; do
   fi
 done
 
-psql -h $PGHOST -p $PGPORT -c "create extension if not exists cube; create extension if not exists earthdistance;" -U $PGUSER $PGDB;
+psql -h $PGHOST -p $PGPORT --file ci/load-psql-extensions.sql -U $PGUSER $PGDB;
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,62 @@
+version: "3.8"
+
+# -----------------------------------------------
+# TODO:
+# * Add redis service
+# * Add a FlexMeasures worker service, maybe using https://docs.docker.com/compose/compose-file/#entrypoint
+# -----------------------------------------------
+
+services:
+  dev-db:
+    image: postgres
+    expose:
+      - 5432
+    restart: always
+    environment:
+      POSTGRES_DB: fm-dev-db
+      POSTGRES_USER: fm-dev-db-user
+      POSTGRES_PASSWORD: fm-dev-db-pass
+    volumes:
+      - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql
+  flexmeasures:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 5000:5000
+    depends_on:
+      - dev-db
+      - test-db  # use -e SQLALCHEMY_TEST_DATABASE_URI=... to exec pytest
+    restart: on-failure
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000/api/v3_0/health/ready"]
+      start_period: 10s
+      interval: 20s
+      timeout: 10s
+      retries: 6
+    environment:
+      SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db"
+      SECRET_KEY: notsecret
+      FLASK_ENV: development 
+    volumes:
+      # a place for config and plugin code - the mount point is for running the FlexMeasures CLI, the 2nd for gunicorn
+      - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro
+      - ./flexmeasures-instance/:/app/instance/:ro
+    command: >
+      bash -c "flexmeasures db upgrade
+      && flexmeasures add toy-account --name 'Docker Toy Account'
+      && gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 2 --threads 4 wsgi:application"
+  test-db:
+    image: postgres
+    expose:
+      - 5432
+    restart: always
+    environment:
+      POSTGRES_DB: fm-test-db
+      POSTGRES_USER: fm-test-db-user
+      POSTGRES_PASSWORD: fm-test-db-pass
+    volumes:
+      - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql
+
+volumes:
+  flexmeasures-instance:
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -9,12 +9,14 @@ v0.10.0 | April XX, 2022
 New features
 -----------
 * Improve legibility of chart axes [see `PR #413 <http://www.github.com/FlexMeasures/flexmeasures/pull/413>`_]
+* API provides health readiness check at /api/v3_0/health/ready [see `PR #416 <http://www.github.com/FlexMeasures/flexmeasures/pull/416>`_]
 
 Bugfixes
 -----------
 
 Infrastructure / Support
 ----------------------
+* Dockerfile to run FlexMeasures API in container; also docker-compose file [see `PR #416 <http://www.github.com/FlexMeasures/flexmeasures/pull/416>`_]
 * Unit conversion prefers shorter units in general [see `PR #415 <http://www.github.com/FlexMeasures/flexmeasures/pull/415>`_]
 * Shorter CI builds in Github Actions by caching Python environment [see `PR #361 <http://www.github.com/FlexMeasures/flexmeasures/pull/361>`_]
 
@@ -36,6 +38,7 @@ Bugfixes
 * Fix filter for selecting one deterministic belief per event, which was duplicating index levels [see `PR #414 <http://www.github.com/FlexMeasures/flexmeasures/pull/414>`_]
 
 
+
 v0.9.1 | March 31, 2022
 ===========================
 

diff --git a/documentation/configuration.rst b/documentation/configuration.rst
@@ -6,7 +6,7 @@ Configuration
 The following configurations are used by FlexMeasures.
 
 Required settings (e.g. postgres db) are marked with a double star (**).
-To enable easier quickstart tutorials, these settings can be set by environment variables.
+To enable easier quickstart tutorials, these required settings can be set by environment variables.
 Recommended settings (e.g. mail, redis) are marked by one star (*).
 
 .. note:: FlexMeasures is best configured via a config file. The config file for FlexMeasures can be placed in one of two locations: 
@@ -302,6 +302,16 @@ Default:
        }
 
 
+SQLALCHEMY_TEST_DATABASE_URI
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When running tests (``make test``, which runs ``pytest``), the default database URI is set in ``utils.config_defaults.TestingConfig``.
+You can use this setting to overwrite that URI and point the tests to an (empty) database of your choice. 
+
+.. note:: This setting is only supported as an environment variable, not in a config file, and only during testing.
+
+
+
 Security
 --------
 

diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst
@@ -0,0 +1,149 @@
+.. docker:
+
+Running via Docker
+======================
+
+FlexMeasures can be run via `docker <https://hub.docker.com/repository/docker/flexmeasures/flexmeasures>`_.
+
+`Docker <https://docs.docker.com/get-docker/>`_ is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner.
+For now, the use case is local development. Using in production is a goal for later.
+
+We also support running all needed parts of a FlexMeasures EMS setup via `docker-compose <https://docs.docker.com/compose/>`_, which is helpful for developers and might inform hosting efforts. 
+
+.. warning:: The dockerization is still `under development <https://github.com/FlexMeasures/flexmeasures/projects/5>`_.
+
+
+The `flexmeasures` image
+-----------------------------------
+
+Getting the image
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use versions we host at Docker Hub, e.g.:
+
+.. code-block:: bash
+
+    docker pull flexmeasures/flexmeasures:latest
+
+
+You can also build the FlexMeasures image yourself, from source:
+
+.. code-block:: bash
+
+    docker build -t flexmeasures/my-version . 
+
+The tag is your choice.
+
+
+Running
+^^^^^^^^^^^
+
+Running the image (as a container) might work like this (remember to get the image first, see above):
+
+.. code-block:: bash
+
+    docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host flexmeasures/flexmeasures
+
+.. note:: Don't know what your image is called (it's "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have.
+
+The two minimal environment variables to run the container successfully are the database URI and the secret key.
+
+In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well.
+
+Browsing ``http://localhost:5000`` should work now and ask you to log in.
+
+Of course, you might not have created a user. You can use ``docker exec -it <flexmeasures-container-name> bash`` to go inside the container and use the :ref:`cli` to create everything you need. 
+
+
+Configuration and customization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables. A good example is the :ref:`mapbox_access_token`, so you can load maps on the dashboard.
+
+To load a configuration file into the container when starting up, we make use of the `instance folder <https://flask.palletsprojects.com/en/2.1.x/config/#instance-folders>`_. You can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this:
+
+.. code-block:: bash
+
+    docker run -v $(pwd)/flexmeasures-instance:/app/instance:ro -d --net=host flexmeasures/flexmeasures
+
+.. warning:: The location of the instance folder depends on how we serve FlexMeasures. The above works with gunicorn. See the compose file for an alternative (for the FlexMeasures CLI), and you can also read the above link about the instance folder.
+
+.. note:: This is also a way to add your custom logic (as described in :ref:`plugins`) to the container. We'll document that shortly. Plugins which should be installed (e.g. by ``pip``) are a bit more difficult to support (you'd need to add `pip install` before the actual entry point). Ideas welcome. 
+
+
+The complete stack: compose
+--------------------------
+
+There are situations, for instance when developing or testing, when you want the whole stack of necessary nodes to be spun up by Docker. `Docker compose <https://docs.docker.com/compose/>`_ is the answer for that.
+
+
+Build the compose stack
+^^^^^^^^^^^^^^^^^
+
+Run this:
+
+.. code-block:: bash
+
+    docker-compose build
+
+This pulls the images you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image.
+
+This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub.
+
+.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418<https://github.com/FlexMeasures/flexmeasures/issues/418>`_).
+
+
+Run the compose stack
+^^^^^^^^^^^^^^^^^^^^^^
+
+Start the stack like this:
+
+.. code-block:: bash
+
+    docker-compose up
+
+You can see log output in the terminal, but ``docker-compose logs`` is also available to you.
+
+Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running:
+
+
+.. code-block:: console
+
+    ± docker ps
+    CONTAINER ID        IMAGE                       COMMAND                  CREATED             STATUS                    PORTS                    NAMES
+    6105f6d1c91f        flexmeasures_flexmeasures   "bash -c 'flexmeasur…"   45 seconds ago      Up 44 seconds (healthy)   0.0.0.0:5000->5000/tcp   flexmeasures_flexmeasures_1
+    b48e4b9b113b        postgres                    "docker-entrypoint.s…"   44 hours ago        Up 45 seconds             5432/tcp                 flexmeasures_dev-db_1
+
+
+The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact.
+
+You can use ``docker-compose logs`` to look at output. ``docker inspect <container>`` and ``docker exec -it <container> bash`` can be quite useful to dive into details. 
+
+.. todo:: We should provide a way to test that this is working, e.g. a list of steps. Document this, but also include that in our tsc/Release list (as a test step to see if Dockerization still works, plus a publish step for the released version).
+
+
+Configuration
+^^^^^^^^^^^^^^
+
+You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``.
+
+
+Data
+^^^^^^
+
+The postgres database is a test database with toy data filled in when the flexmeasures container starts.
+You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. 
+
+
+Running tests
+^^^^^^^^^^^^^^
+
+You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file. Per default, we are using the ``dev-db`` database service.
+
+After you've started the compose stack with ``docker-compose up``, run:
+
+.. code-block:: console
+
+    docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" <flexmeasures-container-name> pytest
+
+This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. 
diff --git a/documentation/host/data.rst b/documentation/host/data.rst
@@ -33,7 +33,7 @@ On Unix:
 
 .. code-block:: console
 
-   sudo apt-get install postgresql
+   sudo apt-get install postgresql-12
    pip install psycopg2-binary
 
 

diff --git a/documentation/host/deployment.rst b/documentation/host/deployment.rst
@@ -5,11 +5,12 @@ How to deploy FlexMeasures
 
 Here you can learn how to get FlexMeasures onto a server.
 
+.. note:: FlexMeasures can be deployed via Docker. Read more at :ref:`docker`.
+
 .. contents:: Table of contents
     :local:
     :depth: 1
 
-.. todo:: It would be great to enable Dockerization of FlexMeasures, let us know if this matters to you.
 
 
 WSGI configuration
@@ -23,9 +24,9 @@ On your own computer, ``flexmeasures run`` is a nice way to start FlexMeasures.
    # web application.
    # It works by setting the variable 'application' to a WSGI handler of some description.
 
+   # use this if you serve from code
    import os
    from dotenv import load_dotenv
-
    project_home = u'/path/to/your/code/flexmeasures'
    load_dotenv(os.path.join(project_home, '.env'))
 

diff --git a/documentation/index.rst b/documentation/index.rst
@@ -23,7 +23,7 @@ A tiny, but complete example: Let's install FlexMeasures from scratch. Then, usi
 
 .. code-block:: console
 
-    $ pip install flexmeasures
+    $ pip install flexmeasures  # also available via Docker
     $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest 
     $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" && export SECRET_KEY=notsecret 
     $ flexmeasures db upgrade  # create tables
@@ -189,6 +189,7 @@ The platform operator of FlexMeasures can be an Aggregator.
     dev/api
     dev/ci
     dev/auth
+    dev/docker
 
 
 

diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst
@@ -10,6 +10,8 @@ Let's walk through an example from scratch! We'll ...
 - load hourly prices
 - optimize a 12h-schedule for a battery that is half full
 
+You'll need: A Unix computer with Python 3.8+, pip and `Docker <https://www.docker.com/>`_.
+
 Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat:
 
 .. code-block:: console
@@ -30,9 +32,9 @@ Okay, let's get started!
 Install Flexmeasures and the database
 ---------------------------------------
 
-This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python and two well-known developer tools, `pip <https://pip.pypa.io>`_ and `docker <https://www.docker.com/>`_
+This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip <https://pip.pypa.io>`_ and `docker <https://www.docker.com/>`_.
 
-We install the FlexMeasures platform, use Docker to run a postgres database and tell FlexMeasures to create all tables.
+We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables.
 
 .. code-block:: console
 
@@ -41,6 +43,8 @@ We install the FlexMeasures platform, use Docker to run a postgres database and
     $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="WARNING" DEBUG=0 
     $ flexmeasures db upgrade
 
+.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker`). We plan to rewrite this tutorial to include steps for users running the complete FlexMeasures platform inside docker containers.
+
 
 Add some structural data
 ---------------------------------------