From 4cb63591912b778e210c1c2b50f48a824a5032f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 25 Apr 2022 13:36:34 +0200 Subject: [PATCH] First version of Dockerfile & docker-compose (#416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * First version of Dockerfile & docker-compose, including a health check in the FM API Signed-off-by: Nicolas Höning * fix inability to call flexmeasures command in container (had to use flask), by not installing in editable mode Signed-off-by: Nicolas Höning * load the instance directory as volume, which allows to pass a configuration file; use flexmeasures run as default command for now Signed-off-by: Nicolas Höning * document better & correctly how to mount config file into a FlexMeasures container, clean up docs Signed-off-by: Nicolas Höning * copy less unneeded files into FlexMeasures image Signed-off-by: Nicolas Höning * run the FlexMeasures image via Gunicorn Signed-off-by: Nicolas Höning * install gunicorn early on Signed-off-by: Nicolas Höning * add changelog entries Signed-off-by: Nicolas Höning * mention the Docker image on index and in toy tutorial section Signed-off-by: Nicolas Höning * move the note about using Docker for the tutorial a little lower Signed-off-by: Nicolas Höning * install solver as well Signed-off-by: Nicolas Höning * document how at this point one can run tests inside a FlexMeasures container Signed-off-by: Nicolas Höning * load sql extensions into postgres service Signed-off-by: Nicolas Höning * add another note to the preliminary testing setup Signed-off-by: Nicolas Höning * Run tests easily in Docker: add test-db service in compose stack, and a way to tell FlexMeasures to use a different database URI than the standard one when testing Signed-off-by: Nicolas Höning * improve documentation, from review comments Signed-off-by: Nicolas Höning * fix typo Signed-off-by: Nicolas Höning * lazy loading in SensorIdField (fixes flexmeasures add schedule) Signed-off-by: Nicolas Höning * docs: add FLASK_ENV=development to docker run call Signed-off-by: Nicolas Höning * split docker and docker-compose docs (first is relevant for installing/deploying), the latter more for development. Also fix small errors and give better info on WSGI scripts. Signed-off-by: Nicolas Höning * fix regression in not attaching account to assets correctly in toy tutorial Signed-off-by: Nicolas Höning * separate installation instructions for toy tutorial into Docker and non-Docker Signed-off-by: Nicolas Höning * some text improvements Signed-off-by: Nicolas Höning * correct the link in coverage badge, to report on main branch Signed-off-by: Nicolas Höning * switch to lfenergy/flexmeasures Signed-off-by: Nicolas Höning * small comments from review Signed-off-by: Nicolas Höning * demanding the lowest docker compose version we know we need (we use start_period for healthchecks) Signed-off-by: Nicolas Höning * more attention-grabbing note about the tutorial happening inside the docker container Signed-off-by: Nicolas Höning * not only the API runs in the container Signed-off-by: Nicolas Höning * better name for the Docker container in the compose tutorial & smaller doc fixes Signed-off-by: Nicolas Höning --- .gitignore | 1 + Dockerfile | 42 ++++++++++ Makefile | 2 +- Readme.md | 4 +- ci/load-psql-extensions.sql | 2 + ci/setup-postgres.sh | 2 +- docker-compose.yml | 63 ++++++++++++++ documentation/changelog.rst | 2 + documentation/conf.py | 5 +- documentation/configuration.rst | 12 ++- documentation/dev/ci.rst | 14 +++- documentation/dev/docker-compose.rst | 83 +++++++++++++++++++ documentation/host/data.rst | 2 +- documentation/host/deployment.rst | 19 +++-- documentation/host/docker.rst | 74 +++++++++++++++++ documentation/index.rst | 8 +- documentation/tut/installation.rst | 14 ++-- .../tut/toy-example-from-scratch.rst | 64 ++++++++++++-- flexmeasures/api/v3_0/__init__.py | 2 + flexmeasures/api/v3_0/health.py | 32 +++++++ flexmeasures/cli/data_add.py | 25 ++++-- flexmeasures/data/schemas/sensors.py | 2 + flexmeasures/utils/config_utils.py | 6 +- requirements/app.txt | 1 - requirements/docs.in | 1 + requirements/docs.txt | 8 +- wsgi.py | 3 + 27 files changed, 451 insertions(+), 42 deletions(-) create mode 100644 Dockerfile create mode 100644 ci/load-psql-extensions.sql create mode 100644 docker-compose.yml create mode 100644 documentation/dev/docker-compose.rst create mode 100644 documentation/host/docker.rst create mode 100644 flexmeasures/api/v3_0/health.py create mode 100644 wsgi.py diff --git a/.gitignore b/.gitignore index 519e917df..38ef57bce 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ venv .env-flexmeasures/ flexmeasures.egg-info flexmeasures.log* +*.csv .cache __pycache__ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..7999d4731 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +FROM ubuntu:focal + +# TODO: Cbc solver +# TODO: run gunicorn as entry command + +ENV DEBIAN_FRONTEND noninteractive +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +# pre-requisites +RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl gunicorn coinor-cbc + +WORKDIR /app +# requirements - doing this earlier, so we don't install them each time. Use --no-cache to refresh them. +COPY requirements /app/requirements + +# py dev tooling +RUN python3 -m pip install --upgrade pip && python3 --version +RUN pip3 install --upgrade setuptools +RUN pip3 install -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt + +# Copy code and meta/config data +COPY setup.* .flaskenv wsgi.py /app/ +COPY flexmeasures/ /app/flexmeasures +RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf +COPY .git/ /app/.git + +RUN pip3 install . + +EXPOSE 5000 + +CMD [ \ + "gunicorn", \ + "--bind", "0.0.0.0:5000", \ + # This is set to /tmp by default, but this is part of the Docker overlay filesystem, and can cause stalls. + # http://docs.gunicorn.org/en/latest/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod + "--worker-tmp-dir", "/dev/shm", \ + # Ideally you'd want one worker per container, but we don't want to risk the health check timing out because + # another request is taking a long time to complete. + "--workers", "2", "--threads", "4", \ + "wsgi:application" \ +] diff --git a/Makefile b/Makefile index 7a1436707..85ae39362 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ update-docs: @echo "Creating docs environment ..." make install-docs-dependencies @echo "Creating documentation ..." - cd documentation; make clean; make html; cd .. + cd documentation; make clean; make html SPHINXOPTS="-W --keep-going -n"; cd .. update-docs-pdf: @echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)" diff --git a/Readme.md b/Readme.md index 514a6abb1..c6c0cd090 100644 --- a/Readme.md +++ b/Readme.md @@ -6,7 +6,7 @@ [![](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Documentation Status](https://readthedocs.org/projects/flexmeasures/badge/?version=latest)](https://flexmeasures.readthedocs.io/en/latest/?badge=latest) -[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg?branch=coverage-in-ci)](https://coveralls.io/github/FlexMeasures/flexmeasures?branch=coverage-in-ci) +[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg)](https://coveralls.io/github/FlexMeasures/flexmeasures) The *FlexMeasures Platform* is the intelligent EMS (energy management system) to support real-time energy flexibility apps, rapidly and scalable. @@ -42,4 +42,4 @@ We made FlexMeasures freely available under the Apache2.0 licence and it is now Within the FlexMeasures project, [we welcome contributions](https://github.com/FlexMeasures/tsc/blob/main/CONTRIBUTING.md). You can also [learn more about our governance](https://github.com/Flexmeasures/tsc/blob/main/GOVERNANCE.md). -You can connect with the community here on Github (e.g. by creating an issue), on [the mailing list](https://lists.lfenergy.org/g/flexmeasures), on [the FlexMeasures channel within the LF Energy Slack](https://slack.lfenergy.org/) or [by contacting the current maintainers](https://www.seita.nl/contact). \ No newline at end of file +You can connect with the community here on Github (e.g. by creating an issue), on [the mailing list](https://lists.lfenergy.org/g/flexmeasures), on [the FlexMeasures channel within the LF Energy Slack](https://slack.lfenergy.org/) or [by contacting the current maintainers](https://www.seita.nl/contact). diff --git a/ci/load-psql-extensions.sql b/ci/load-psql-extensions.sql new file mode 100644 index 000000000..3e1526617 --- /dev/null +++ b/ci/load-psql-extensions.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS cube; +CREATE EXTENSION IF NOT EXISTS earthdistance; diff --git a/ci/setup-postgres.sh b/ci/setup-postgres.sh index 98b6af82a..0531b4b90 100755 --- a/ci/setup-postgres.sh +++ b/ci/setup-postgres.sh @@ -27,4 +27,4 @@ while [[ true ]]; do fi done -psql -h $PGHOST -p $PGPORT -c "create extension if not exists cube; create extension if not exists earthdistance;" -U $PGUSER $PGDB; +psql -h $PGHOST -p $PGPORT --file ci/load-psql-extensions.sql -U $PGUSER $PGDB; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..5c6862e07 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +version: "3.4" + +# ----------------------------------------------- +# TODO: +# * Add redis service +# * Add a FlexMeasures worker service, maybe using https://docs.docker.com/compose/compose-file/#entrypoint +# ----------------------------------------------- + +services: + dev-db: + image: postgres + expose: + - 5432 + restart: always + environment: + POSTGRES_DB: fm-dev-db + POSTGRES_USER: fm-dev-db-user + POSTGRES_PASSWORD: fm-dev-db-pass + volumes: + - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql + server: + build: + context: . + dockerfile: Dockerfile + ports: + - 5000:5000 + depends_on: + - dev-db + - test-db # use -e SQLALCHEMY_TEST_DATABASE_URI=... to exec pytest + restart: on-failure + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/api/v3_0/health/ready"] + start_period: 10s + interval: 20s + timeout: 10s + retries: 6 + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" + SECRET_KEY: notsecret + FLASK_ENV: development + LOGGING_LEVEL: INFO + volumes: + # a place for config and plugin code - the mount point is for running the FlexMeasures CLI, the 2nd for gunicorn + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro + - ./flexmeasures-instance/:/app/instance/:ro + command: > + bash -c "flexmeasures db upgrade + && flexmeasures add toy-account --name 'Docker Toy Account' + && gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 2 --threads 4 wsgi:application" + test-db: + image: postgres + expose: + - 5432 + restart: always + environment: + POSTGRES_DB: fm-test-db + POSTGRES_USER: fm-test-db-user + POSTGRES_PASSWORD: fm-test-db-pass + volumes: + - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql + +volumes: + flexmeasures-instance: \ No newline at end of file diff --git a/documentation/changelog.rst b/documentation/changelog.rst index d29556cb5..02cbf1fb0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,12 +9,14 @@ v0.10.0 | April XX, 2022 New features ----------- * Improve legibility of chart axes [see `PR #413 `_] +* API provides health readiness check at /api/v3_0/health/ready [see `PR #416 `_] Bugfixes ----------- Infrastructure / Support ---------------------- +* Dockerfile to run FlexMeasures in container; also docker-compose file [see `PR #416 `_] * Unit conversion prefers shorter units in general [see `PR #415 `_] * Shorter CI builds in Github Actions by caching Python environment [see `PR #361 `_] * Allow to filter data by source using a tuple instead of a list [see `PR #421 `_] diff --git a/documentation/conf.py b/documentation/conf.py index 763a72084..961df47c6 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -48,6 +48,7 @@ "sphinx.ext.ifconfig", "sphinx.ext.todo", "sphinx_copybutton", + "sphinx_tabs.tabs", "sphinx_fontawesome", "sphinxcontrib.autohttp.flask", "sphinxcontrib.autohttp.flaskqref", @@ -77,7 +78,7 @@ # This pattern also affects html_static_path and html_extra_path . exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# Todo: these are not mature enough yet for release +# Todo: these are not mature enough yet for release, or should be removed exclude_patterns.append("int/*.rst") exclude_patterns.append("concepts/assets.rst") exclude_patterns.append("concepts/markets.rst") @@ -194,7 +195,7 @@ # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} +intersphinx_mapping = {"https://docs.python.org/3/": None} # -- Options for copybytton extension --------------------------------------- copybutton_prompt_is_regexp = True diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 32c3129cd..c8bda48d5 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -6,7 +6,7 @@ Configuration The following configurations are used by FlexMeasures. Required settings (e.g. postgres db) are marked with a double star (**). -To enable easier quickstart tutorials, these settings can be set by environment variables. +To enable easier quickstart tutorials, these required settings can be set by environment variables. Recommended settings (e.g. mail, redis) are marked by one star (*). .. note:: FlexMeasures is best configured via a config file. The config file for FlexMeasures can be placed in one of two locations: @@ -302,6 +302,16 @@ Default: } +SQLALCHEMY_TEST_DATABASE_URI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When running tests (``make test``, which runs ``pytest``), the default database URI is set in ``utils.config_defaults.TestingConfig``. +You can use this setting to overwrite that URI and point the tests to an (empty) database of your choice. + +.. note:: This setting is only supported as an environment variable, not in a config file, and only during testing. + + + Security -------- diff --git a/documentation/dev/ci.rst b/documentation/dev/ci.rst index 715e27aca..222f27a9a 100644 --- a/documentation/dev/ci.rst +++ b/documentation/dev/ci.rst @@ -1,4 +1,4 @@ -.. continuous_integration: +.. _continuous_integration: Continuous integration ====================== @@ -97,3 +97,15 @@ The last step, touching a wsgi.py file, is often used as a way to soft-restart t echo "RESTARTING APPLICATION ..." touch $PATH_TO_WSGI + + +A WSGI file can do various things, as well, but the simplest form is shown below. + +.. code-block:: python + + from flexmeasures.app import create as create_app + + application = create_app() + + +The web server is told about the WSGI script, but also about the object which represents the application. For instance, if this script is called ``wsgi.py``, then the relevant argument to the gunicorn server is ``wsgi:application``. \ No newline at end of file diff --git a/documentation/dev/docker-compose.rst b/documentation/dev/docker-compose.rst new file mode 100644 index 000000000..6055df03e --- /dev/null +++ b/documentation/dev/docker-compose.rst @@ -0,0 +1,83 @@ +.. _docker-compose: + +Running a complete stack with docker-compose +============================================= + +To install FlexMeasures, plus the libraries and databases it depends on, on your computer is some work, and can have unexpected hurdles, e.g. depending on the operating system. A nice alternative is to let that happen within Docker. The whole stack can be run via `Docker compose `_, saving the developer much time. + +For this, we assume you are in the directory housing ``docker-compose.yml``. + + +.. note:: The minimum Docker version is 17.09 and for docker-compose we tested successfully at version 1.25. You can check your versions with ``docker[-compose] --version``. + +Build the compose stack +------------------------ + +Run this: + +.. code-block:: bash + + docker-compose build + +This pulls the images you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. + +This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub. + +.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418 `_). + + +Run the compose stack +---------------------- + +Start the stack like this: + +.. code-block:: bash + + docker-compose up + +You can see log output in the terminal, but ``docker-compose logs`` is also available to you. + +Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running: + + +.. code-block:: console + + ± docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + dda1a8606926 flexmeasures_server "bash -c 'flexmeasur…" 43 seconds ago Up 41 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_server_1 + 27ed9eef1b04 postgres "docker-entrypoint.s…" 2 days ago Up 42 seconds 5432/tcp flexmeasures_dev-db_1 + 90df2065e08d postgres "docker-entrypoint.s…" 2 days ago Up 42 seconds 5432/tcp flexmeasures_test-db_1 + + +The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact. + +You can use ``docker-compose logs`` to look at output. ``docker inspect `` and ``docker exec -it bash`` can be quite useful to dive into details. + +.. todo:: We should provide a way to test that this is working, e.g. a list of steps. Document this, but also include that in our tsc/Release list (as a test step to see if Dockerization still works, plus a publish step for the released version). + + +Configuration +--------------- + +You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``. + + +Data +------- + +The postgres database is a test database with toy data filled in when the flexmeasures container starts. +You could also connect it to some other database, by setting a different ``SQLALCHEMY_DATABASE_URI`` in the config. + + +Running tests +--------------- + +You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file (per default, we are using the ``dev-db`` database service). + +After you've started the compose stack with ``docker-compose up``, run: + +.. code-block:: console + + docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" flexmeasures_server_1 pytest + +This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. \ No newline at end of file diff --git a/documentation/host/data.rst b/documentation/host/data.rst index cb6c00ec9..f40c688dc 100644 --- a/documentation/host/data.rst +++ b/documentation/host/data.rst @@ -33,7 +33,7 @@ On Unix: .. code-block:: console - sudo apt-get install postgresql + sudo apt-get install postgresql-12 pip install psycopg2-binary diff --git a/documentation/host/deployment.rst b/documentation/host/deployment.rst index ef791f97b..4306bae16 100644 --- a/documentation/host/deployment.rst +++ b/documentation/host/deployment.rst @@ -5,11 +5,12 @@ How to deploy FlexMeasures Here you can learn how to get FlexMeasures onto a server. +.. note:: FlexMeasures can be deployed via Docker. Read more at :ref:`docker-image`. + .. contents:: Table of contents :local: :depth: 1 -.. todo:: It would be great to enable Dockerization of FlexMeasures, let us know if this matters to you. WSGI configuration @@ -22,17 +23,25 @@ On your own computer, ``flexmeasures run`` is a nice way to start FlexMeasures. # This file contains the WSGI configuration required to serve up your # web application. # It works by setting the variable 'application' to a WSGI handler of some description. + # The crucial part are the last two lines. We add some ideas for possible other logic. import os - from dotenv import load_dotenv - project_home = u'/path/to/your/code/flexmeasures' + # use this if you want to load your own ``.env`` file. + from dotenv import load_dotenv load_dotenv(os.path.join(project_home, '.env')) + # use this if you run from source + if project_home not in sys.path: + sys.path = [project_home] + sys.path + # adapt PATH to find our LP solver if it is installed from source + os.environ["PATH"] = os.environ.get("PATH") + ":/home/seita/Cbc-2.9/bin" - # create flask app - need to call it "application" for WSGI to work + # create flask app - the name "application" has to be passed to the WSGI server from flexmeasures.app import create as create_app application = create_app() +The web server is told about the WSGI script, but also about the object which represents the application. For instance, if this script is called ``wsgi.py``, then the relevant argument to the gunicorn server is ``wsgi:application``. + Keep in mind that FlexMeasures is based on `Flask `_, so almost all knowledge on the web on how to deploy a Flask app also helps with deploying FlexMeasures. @@ -52,7 +61,7 @@ You can install it on Debian like this: If you can't use the package manager on your host, the solver has to be installed from source. -We provide `an example script `_ to do that, where you can also +We provide an example script in ``ci/install-cbc-from-source.sh`` to do that, where you can also pass a directory for the installation. In case you want to install a later version, adapt the version in the script. diff --git a/documentation/host/docker.rst b/documentation/host/docker.rst new file mode 100644 index 000000000..730499558 --- /dev/null +++ b/documentation/host/docker.rst @@ -0,0 +1,74 @@ +.. _docker-image: + +Running via Docker +====================== + +FlexMeasures can be run via `docker `_. + +`Docker `_ is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. +For now, the use case is local development. Using in production is a goal for later. + +We also support running all needed parts of a FlexMeasures EMS setup via `docker-compose `_, which is helpful for developers and might inform hosting efforts. + +.. warning:: The dockerization is still `under development `_. + +We also provide a docker-compose file for development, see :ref:`docker-compose`. + + +The `flexmeasures` image +----------------------------------- + +Getting the image +^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use versions we host at Docker Hub, e.g.: + +.. code-block:: bash + + docker pull lfenergy/flexmeasures:latest + + +You can also build the FlexMeasures image yourself, from source: + +.. code-block:: bash + + docker build -t flexmeasures/my-version . + +The tag is your choice. + + +Running +^^^^^^^^^^^ + +Running the image (as a container) might work like this (remember to get the image first, see above): + +.. code-block:: bash + + docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla --env FLASK_ENV=development -d --net=host lfenergy/flexmeasures + +.. note:: Don't know what your image is called (its "tag")? We used ``lfenergy/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. + +The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``FLASK_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). If you see too much output, you can also set ``LOGGING_LEVEL=INFO``. + +In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well. + +Browsing ``http://localhost:5000`` should work now and ask you to log in. + +Of course, you might not have created a user. You can use ``docker exec -it bash`` to go inside the container and use the :ref:`cli` to create everything you need. + + +Configuration and customization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables. A good example is the :ref:`mapbox_access_token`, so you can load maps on the dashboard. + +To load a configuration file into the container when starting up, we make use of the `instance folder `_. You can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: + +.. code-block:: bash + + docker run -v $(pwd)/flexmeasures-instance:/app/instance:ro -d --net=host lfenergy/flexmeasures + +.. warning:: The location of the instance folder depends on how we serve FlexMeasures. The above works with gunicorn. See the compose file for an alternative (for the FlexMeasures CLI), and you can also read the above link about the instance folder. + +.. note:: This is also a way to add your custom logic (as described in :ref:`plugins`) to the container. We'll document that shortly. Plugins which should be installed (e.g. by ``pip``) are a bit more difficult to support (you'd need to add `pip install` before the actual entry point). Ideas welcome. + diff --git a/documentation/index.rst b/documentation/index.rst index cdeab0919..8775b1baa 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -23,7 +23,7 @@ A tiny, but complete example: Let's install FlexMeasures from scratch. Then, usi .. code-block:: console - $ pip install flexmeasures + $ pip install flexmeasures # also available via Docker $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" && export SECRET_KEY=notsecret $ flexmeasures db upgrade # create tables @@ -44,7 +44,7 @@ As possible users, we see energy service companies (ESCOs) who want to build rea However, even small companies and hobby projects might find FlexMeasures useful! We are constantly improving the ease of use. -FlexMeasures can be used as your EMS, but is also to integrate with existing systems as a smart backend or add-on to deal with energy flexibility specifically. +FlexMeasures can be used as your EMS, but it can also integrate with existing systems as a smart backend, or as an add-on to deal with energy flexibility specifically. The image below shows how FlexMeasures, with the help of plugins fitted for a given use case, turns data into optimized schedules: @@ -165,8 +165,9 @@ The platform operator of FlexMeasures can be an Aggregator. :caption: Hosting FlexMeasures :maxdepth: 1 - host/deployment + host/docker host/data + host/deployment host/error-monitoring host/modes @@ -189,6 +190,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/api dev/ci dev/auth + dev/docker-compose diff --git a/documentation/tut/installation.rst b/documentation/tut/installation.rst index 0888a0ccd..0d5f94ec2 100644 --- a/documentation/tut/installation.rst +++ b/documentation/tut/installation.rst @@ -6,9 +6,14 @@ Installation & First steps Getting FlexMeasures to run ----------------------------- -This section walks you through getting FlexMeasures to run. For an example with the least effort, see :ref:`tut_toy_schedule`. Here, we'll cover getting started with an installation you run continuously ― making a secret key, connecting a database and creating one user & one asset. +This section walks you through installing FlexMeasures on your own PC and running it continuously. +We'll cover getting started by making a secret key, connecting a database and creating one user & one asset. -.. note:: Are you not hosting FlexMeasures, but want to learn how to interact with it? Start with :ref:`tut_posting_data`. +.. note:: Maybe these starting points are also interesting for you: + + * For an example to see FlexMeasures in action with the least effort, see :ref:`tut_toy_schedule`. + * You can run FlexMeasures via Docker, see :ref:`docker` and :ref:`docker-compose`. + * Are you not hosting FlexMeasures, but want to learn how to interact with it? Start with :ref:`tut_posting_data`. Install FlexMeasures @@ -205,14 +210,13 @@ It's finally time to start running FlexMeasures: (This might print some warnings, see the next section where we go into more detail) .. note:: In a production context, you shouldn't run a script - hand the ``app`` object to a WSGI process, as your platform of choice describes. - Often, that requires a WSGI script. We provide an example WSGI script in :ref:`continuous_integration`. + Often, that requires a WSGI script. We provide an example WSGI script in :ref:`continuous_integration`. You can also take a look at FlexMeasures' Dockerfile to get an idea how to run FlexMeasures with gunicorn. You can visit ``http://localhost:5000`` now to see if the app's UI works. When you see the dashboard, the map will not work. For that, you'll need to get your :ref:`mapbox_access_token` and add it to your config file. - Other settings, for full functionality -------------------------------------- @@ -258,4 +262,4 @@ Where to go from here? If your data structure is good, you should think about (continually) adding measurement data. This tutorial mentioned how to add data, but :ref:`_tut_posting_data` goes deeper with examples and terms & definitions. -Then, you probably want to use FlexMeasures to generate forecasts and schedules! For this, read further in :ref:`_tut_forecasting_scheduling`. \ No newline at end of file +Then, you probably want to use FlexMeasures to generate forecasts and schedules! For this, read further in :ref:`tut_forecasting_scheduling`. \ No newline at end of file diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 22b24c743..031df4e97 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -10,6 +10,8 @@ Let's walk through an example from scratch! We'll ... - load hourly prices - optimize a 12h-schedule for a battery that is half full +What do you need? Your own computer, with one of two situations: Either you have `Docker `_ or your computer supports Python 3.8+, pip and PostgresDB. The former might be easier, see the installation step below. But you choose. + Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat: .. code-block:: console @@ -27,19 +29,65 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s Okay, let's get started! +.. note:: You can copy the commands by hovering on the top right corner of code examples. You'll copy only the commands, not the output! + Install Flexmeasures and the database --------------------------------------- -This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python and two well-known developer tools, `pip `_ and `docker `_ +.. tabs:: -We install the FlexMeasures platform, use Docker to run a postgres database and tell FlexMeasures to create all tables. + .. tab:: Docker -.. code-block:: console + If `docker `_ is running on your system, you're good to go. Otherwise, see `here `_. + + We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. + + .. code-block:: console + + $ docker pull lfenergy/flexmeasures:latest + $ docker pull postgres + $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=fm-db-passwd -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest + $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:fm-db-passwd@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host lfenergy/flexmeasures + $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade" + + Now - what's *very important* to remember is this: The rest of this tutorial will happen *inside* the ``flexmeasures-tutorial-fm`` container! This is how you hop inside the container and run a terminal there: + + .. code-block:: console + + $ docker exec -it flexmeasures-tutorial-fm bash + + To leave the container session, hold CTRL-C or type "exit". + + To stop the containers, you can type + + .. code-block:: console + + $ docker stop flexmeasures-tutorial-db + $ docker stop flexmeasures-tutorial-fm + + + .. tab:: On your PC + + This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `postgres `_. + + We'll create a database for FlexMeasures: + + .. code-block:: console + + sudo -i -u postgres + createdb -U postgres flexmeasures-db + createuser --pwprompt -U postgres flexmeasures-user # enter your password, we'll use "fm-db-passwd" + exit + + Then, we can install FlexMeasures itself, set some variables and tell FlexMeasures to create all tables: + + .. code-block:: console + + $ pip install flexmeasures + $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures-user:fm-db-passwd@localhost:5432/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0 + $ flexmeasures db upgrade - $ pip install flexmeasures - $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest - $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="WARNING" DEBUG=0 - $ flexmeasures db upgrade + .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which uses a prepared Linux image, so it'll definitely work. Add some structural data @@ -87,7 +135,7 @@ If you want, you can inspect what you created: 2 toy-building building (52.374, 4.88969) 1 toy-solar solar (52.374, 4.88969) - $ flexmeasures show asset --id + $ flexmeasures show asset --id 3 =========================== Asset toy-battery (ID:3): diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index f36281a26..56bc4d840 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -3,6 +3,7 @@ from flexmeasures.api.v3_0.sensors import SensorAPI from flexmeasures.api.v3_0.users import UserAPI from flexmeasures.api.v3_0.assets import AssetAPI +from flexmeasures.api.v3_0.health import HealthAPI def register_at(app: Flask): @@ -13,3 +14,4 @@ def register_at(app: Flask): SensorAPI.register(app, route_prefix=v3_0_api_prefix) UserAPI.register(app, route_prefix=v3_0_api_prefix) AssetAPI.register(app, route_prefix=v3_0_api_prefix) + HealthAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/health.py b/flexmeasures/api/v3_0/health.py new file mode 100644 index 000000000..a3f783496 --- /dev/null +++ b/flexmeasures/api/v3_0/health.py @@ -0,0 +1,32 @@ +from flask import current_app +from flask_classful import FlaskView, route +from flask_json import as_json + +from flexmeasures.data import db + + +def _check_sql_database(): + try: + db.session.execute("SELECT 1").first() + return True + except Exception: # noqa: B902 + current_app.logger.exception("Database down or undetected") + return False + + +class HealthAPI(FlaskView): + + route_base = "/health" + trailing_slash = False + + @route("/ready", methods=["GET"]) + @as_json + def is_ready(self): + """ + Get readiness status + """ + status = {"database_sql": _check_sql_database()} # TODO: check redis + if all(status.values()): + return status, 200 + else: + return status, 503 diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 20b581b55..0eeaa7868 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -937,16 +937,23 @@ def add_toy_account(kind: str, name: str): # make an account (if not exist) account = Account.query.filter(Account.name == name).one_or_none() if account: - click.echo(f"Account {name} already exists. Aborting ...") - raise click.Abort() + click.echo(f"Account {name} already exists.") + return # make an account user (account-admin?) - user = create_user( - email="toy-user@flexmeasures.io", - check_email_deliverability=False, - password="toy-password", - user_roles=["account-admin"], - account_name=name, - ) + email = "toy-user@flexmeasures.io" + user = User.query.filter_by(email=email).one_or_none() + if user is not None: + click.echo( + f"User with email {email} already exists in account {user.account.name}." + ) + else: + user = create_user( + email=email, + check_email_deliverability=False, + password="toy-password", + user_roles=["account-admin"], + account_name=name, + ) # make assets for asset_type in ("solar", "building", "battery"): asset = GenericAsset( diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 6f96cbcae..4a06f306d 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -65,6 +65,8 @@ def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: sensor = Sensor.query.get(value) if sensor is None: raise FMValidationError(f"No sensor found with id {value}.") + # lazy loading now (sensor is somehow not in session after this) + sensor.generic_asset return sensor def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: diff --git a/flexmeasures/utils/config_utils.py b/flexmeasures/utils/config_utils.py index 4a7d28acb..a786b27ad 100644 --- a/flexmeasures/utils/config_utils.py +++ b/flexmeasures/utils/config_utils.py @@ -78,12 +78,16 @@ def read_config(app: Flask, custom_path_to_config: Optional[str]): path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") - # Don't overwrite when testing (that should run completely on defaults) + # Custom config: not when testing (that should run completely on defaults) if not app.testing: used_path_to_config = read_custom_config( app, custom_path_to_config, path_to_config_home, path_to_config_instance ) read_env_vars(app) + else: # one exception: the ability to set where the test database is + custom_test_db_uri = os.getenv("SQLALCHEMY_TEST_DATABASE_URI", None) + if custom_test_db_uri: + app.config["SQLALCHEMY_DATABASE_URI"] = custom_test_db_uri # Check for missing values. # Documentation runs fine without them. diff --git a/requirements/app.txt b/requirements/app.txt index 87534b8ee..08af442c9 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -219,7 +219,6 @@ packaging==21.3 # via # bokeh # matplotlib - # pint # redis # statsmodels # webargs diff --git a/requirements/docs.in b/requirements/docs.in index 96f695783..49cc0f542 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -5,3 +5,4 @@ sphinx-rtd-theme sphinxcontrib.httpdomain sphinx_fontawesome sphinx_copybutton +sphinx-tabs diff --git a/requirements/docs.txt b/requirements/docs.txt index 17ed18975..d830d8497 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -32,6 +32,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme + # sphinx-tabs idna==3.3 # via # -c requirements/app.txt @@ -60,7 +61,9 @@ pycparser==2.21 # -c requirements/app.txt # cffi pygments==2.11.2 - # via sphinx + # via + # sphinx + # sphinx-tabs pyopenssl==22.0.0 # via # -c requirements/app.txt @@ -89,6 +92,7 @@ sphinx==4.4.0 # sphinx-copybutton # sphinx-fontawesome # sphinx-rtd-theme + # sphinx-tabs # sphinxcontrib.httpdomain sphinx-copybutton==0.5.0 # via -r requirements/docs.in @@ -96,6 +100,8 @@ sphinx-fontawesome==0.0.6 # via -r requirements/docs.in sphinx-rtd-theme==1.0.0 # via -r requirements/docs.in +sphinx-tabs==3.3.1 + # via -r requirements/docs.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 000000000..4da29c483 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,3 @@ +from flexmeasures.app import create as create_app + +application = create_app()