From a973344d4acdf44d7666dd93e0124ab441abeff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 10 Apr 2022 17:49:29 +0200 Subject: [PATCH 01/30] First version of Dockerfile & docker-compose, including a health check in the FM API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 40 ++++++++++++++++ docker-compose.yml | 48 +++++++++++++++++++ documentation/dev/docker.rst | 76 +++++++++++++++++++++++++++++++ documentation/host/data.rst | 2 +- documentation/host/deployment.rst | 5 +- documentation/index.rst | 1 + flexmeasures/api/v3_0/__init__.py | 2 + flexmeasures/api/v3_0/health.py | 32 +++++++++++++ flexmeasures/cli/data_add.py | 27 +++++++---- 9 files changed, 220 insertions(+), 13 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 documentation/dev/docker.rst create mode 100644 flexmeasures/api/v3_0/health.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..0151bdd69 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM ubuntu:focal + +# TODO: Cbc solver +# TODO: run gunicorn as entry command + +ENV DEBIAN_FRONTEND noninteractive +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +# pre-requisites +RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl + +WORKDIR /app +# requirements - doing this earlier, so we don't install them each time. Use --no-cache to refresh them. +COPY requirements /app/requirements + +# py dev tooling +RUN python3 -m pip install --upgrade pip && python3 --version +RUN pip3 install --upgrade setuptools +RUN pip3 install -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt + +# Copy code and meta/config data +COPY setup.* .flaskenv .env /app/ +COPY flexmeasures/ /app/flexmeasures +COPY .git/ /app/.git + +RUN pip3 install -e . + +EXPOSE 5000 + +#CMD [ \ +# "gunicorn", flexmeasures.wsgi", \ +# "--bind", "0.0.0.0:5000", \ +# # This is set to /tmp by default, but this is part of the Docker overlay filesystem, and can cause stalls. +# # http://docs.gunicorn.org/en/latest/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod +# "--worker-tmp-dir", "/dev/shm", \ +# # Ideally you'd want one worker per container, but we don't want to risk the health check timing out because +# # another request is taking a long time to complete. +# "--workers", "2", "--threads", "4" \ +#] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..ae2f29ee0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +version: "3.8" + +# ----------------------------------------------- +# TODO: +# * Add redis service +# * Add a worker service, maybe using https://docs.docker.com/compose/compose-file/#entrypoint +# ----------------------------------------------- + +services: + dev-db: + # This toy db is used with SQLALCHEMY_DATABASE_URI=postgresql://fm-dev-db-user:fm-dev-db-pass@localhost:5433/fm-dev-db + # You can set a different SQLALCHEMY_DATABASE_URI in .env, to work on a different database! + image: postgres + expose: + - 5432 + restart: always + environment: + POSTGRES_DB: fm-dev-db + POSTGRES_USER: fm-dev-db-user + POSTGRES_PASSWORD: fm-dev-db-pass + #volumes: + # - ./fm-dev.sql:/docker-entrypoint-initdb.d/fm-dev.sql + flexmeasures: + build: + context: . + dockerfile: Dockerfile + #expose: + # - 5000 + ports: + - 5000:5000 + depends_on: + - dev-db + restart: on-failure + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/api/v3_0/health/ready"] + start_period: 30s + interval: 20s + timeout: 10s + retries: 6 + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" + SECRET_KEY: notsecret + #FLASK_ENV: production + # not sure why, but using flexmeasures receives a "importlib.metadata.PackageNotFoundError". This is a TODO. + command: > + bash -c "flask db upgrade + && flask add toy-account --name 'Docker Toy Account' + && flask run --host 0.0.0.0 --port 5000" diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst new file mode 100644 index 000000000..1acf4521d --- /dev/null +++ b/documentation/dev/docker.rst @@ -0,0 +1,76 @@ +.. docker: + +Running via Docker +====================== + +FlexMeasures can be run via `docker `_. TODO: link to the image once it's up. + +Docker is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. +For now, the use case is local development. Using in production is a goal for later. + +We also support running all needed parts of a FlexMeasures EMS setup via `docker-compose `_, which is helpful for developers and might inform hosting efforts. + +.. warning:: The dockerization is still under development. + +TODO: + +- Ability to load config file into container (flexmeasures.cfg, if available) +- Main Dockerfile serves API (via gunicorn and a WSGI file) +- Compose script defines one additional FM node, where it runs a worker as entry point instead. Document. +- Some way to test that this is working, e.g. a list of steps. Document. Also include in Release list. Could be a test step, then a publish step. +- Fix: flask->flexmeasures (importlib error) +- Publish one flexmeasures image per version + + +Download and run the default image +----------------------------------- + +TODO + + +Build the compose stack +-------------------------- + +Run this: + + docker-compose build + +This builds the containers you need from code. If you change code, re-running this will re-build that image. + +.. note:: Of course the ``pip install`` step takes time - maybe we want to try using pip install and caching: https://medium.com/@scythargon/cache-for-python-pip-downloads-and-wheels-in-docker-67f24e7cd84e) + + +Run the compose stack +------------------ + +Start the stack like this: + + docker-compose up + +You can see log output in the terminal, but ``docker-compose logs`` is also available to you. + +Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running and ``docker-compose logs`` to look at output. ```docker inspect `` can be quite useful to dive into details. + +The FlexMeasures container has a health check implemented which is reflected in this output and you can see which ports are available on your machine to interact. + + +Inspect individual containers +------------------------------- + +To start a bash session in the `flexmeasures` container, do this: + + docker-compose run flexmeasures bash + + +Configuration +---------------- + +You can pass in your own configuration (e.g. MapBox access token, or db URI, see below) like this: TODO + + +Data +----- + +The postgres database is a test database with toy data filled in when the flexmeasures container starts. +You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. + diff --git a/documentation/host/data.rst b/documentation/host/data.rst index cb6c00ec9..f40c688dc 100644 --- a/documentation/host/data.rst +++ b/documentation/host/data.rst @@ -33,7 +33,7 @@ On Unix: .. code-block:: console - sudo apt-get install postgresql + sudo apt-get install postgresql-12 pip install psycopg2-binary diff --git a/documentation/host/deployment.rst b/documentation/host/deployment.rst index ef791f97b..55c4e6a03 100644 --- a/documentation/host/deployment.rst +++ b/documentation/host/deployment.rst @@ -5,11 +5,12 @@ How to deploy FlexMeasures Here you can learn how to get FlexMeasures onto a server. +.. note:: FlexMeasures can be deployed via Docker. Read more at :ref:`docker`. + .. contents:: Table of contents :local: :depth: 1 -.. todo:: It would be great to enable Dockerization of FlexMeasures, let us know if this matters to you. WSGI configuration @@ -23,9 +24,9 @@ On your own computer, ``flexmeasures run`` is a nice way to start FlexMeasures. # web application. # It works by setting the variable 'application' to a WSGI handler of some description. + # use this if you serve from code import os from dotenv import load_dotenv - project_home = u'/path/to/your/code/flexmeasures' load_dotenv(os.path.join(project_home, '.env')) diff --git a/documentation/index.rst b/documentation/index.rst index c90e2c795..bdcee6318 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -180,6 +180,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/api dev/ci dev/auth + dev/docker diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index f36281a26..56bc4d840 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -3,6 +3,7 @@ from flexmeasures.api.v3_0.sensors import SensorAPI from flexmeasures.api.v3_0.users import UserAPI from flexmeasures.api.v3_0.assets import AssetAPI +from flexmeasures.api.v3_0.health import HealthAPI def register_at(app: Flask): @@ -13,3 +14,4 @@ def register_at(app: Flask): SensorAPI.register(app, route_prefix=v3_0_api_prefix) UserAPI.register(app, route_prefix=v3_0_api_prefix) AssetAPI.register(app, route_prefix=v3_0_api_prefix) + HealthAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/health.py b/flexmeasures/api/v3_0/health.py new file mode 100644 index 000000000..a3f783496 --- /dev/null +++ b/flexmeasures/api/v3_0/health.py @@ -0,0 +1,32 @@ +from flask import current_app +from flask_classful import FlaskView, route +from flask_json import as_json + +from flexmeasures.data import db + + +def _check_sql_database(): + try: + db.session.execute("SELECT 1").first() + return True + except Exception: # noqa: B902 + current_app.logger.exception("Database down or undetected") + return False + + +class HealthAPI(FlaskView): + + route_base = "/health" + trailing_slash = False + + @route("/ready", methods=["GET"]) + @as_json + def is_ready(self): + """ + Get readiness status + """ + status = {"database_sql": _check_sql_database()} # TODO: check redis + if all(status.values()): + return status, 200 + else: + return status, 503 diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 20b581b55..828e38e06 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -937,22 +937,29 @@ def add_toy_account(kind: str, name: str): # make an account (if not exist) account = Account.query.filter(Account.name == name).one_or_none() if account: - click.echo(f"Account {name} already exists. Aborting ...") - raise click.Abort() + click.echo(f"Account {name} already exists.") + return # make an account user (account-admin?) - user = create_user( - email="toy-user@flexmeasures.io", - check_email_deliverability=False, - password="toy-password", - user_roles=["account-admin"], - account_name=name, - ) + email = "toy-user@flexmeasures.io" + user = User.query.filter_by(email=email).one_or_none() + if user is not None: + print( + f"User with email {email} already exists in account {user.account.name}." + ) + else: + user = create_user( + email=email, + check_email_deliverability=False, + password="toy-password", + user_roles=["account-admin"], + account_name=name, + ) # make assets for asset_type in ("solar", "building", "battery"): asset = GenericAsset( name=f"toy-{asset_type}", generic_asset_type=asset_types[asset_type], - owner=user.account, + owner=account, latitude=location[0], longitude=location[1], ) From 9ed8213c99c91182d27f45762993c851445e643e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 11 Apr 2022 16:01:36 +0200 Subject: [PATCH 02/30] fix inability to call flexmeasures command in container (had to use flask), by not installing in editable mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 2 +- docker-compose.yml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0151bdd69..93e257d3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY setup.* .flaskenv .env /app/ COPY flexmeasures/ /app/flexmeasures COPY .git/ /app/.git -RUN pip3 install -e . +RUN pip3 install . EXPOSE 5000 diff --git a/docker-compose.yml b/docker-compose.yml index ae2f29ee0..385395695 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,16 +33,15 @@ services: restart: on-failure healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5000/api/v3_0/health/ready"] - start_period: 30s + start_period: 10s interval: 20s timeout: 10s retries: 6 environment: SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" SECRET_KEY: notsecret - #FLASK_ENV: production - # not sure why, but using flexmeasures receives a "importlib.metadata.PackageNotFoundError". This is a TODO. + #FLASK_ENV: production command: > - bash -c "flask db upgrade - && flask add toy-account --name 'Docker Toy Account' - && flask run --host 0.0.0.0 --port 5000" + bash -c "flexmeasures db upgrade + && flexmeasures add toy-account --name 'Docker Toy Account' + && flexmeasures run --host 0.0.0.0 --port 5000" From 0f3dc8d0f45596bd08988df0378d342302edcdbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 12 Apr 2022 00:51:14 +0200 Subject: [PATCH 03/30] load the instance directory as volume, which allows to pass a configuration file; use flexmeasures run as default command for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 2 ++ docker-compose.yml | 9 ++++--- documentation/dev/docker.rst | 48 ++++++++++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93e257d3d..2eac03095 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,8 @@ RUN pip3 install . EXPOSE 5000 +CMD ["flexmeasures", "run", "--host", "0.0.0.0", "--port", "5000"] + #CMD [ \ # "gunicorn", flexmeasures.wsgi", \ # "--bind", "0.0.0.0:5000", \ diff --git a/docker-compose.yml b/docker-compose.yml index 385395695..e15bc12ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,8 +24,6 @@ services: build: context: . dockerfile: Dockerfile - #expose: - # - 5000 ports: - 5000:5000 depends_on: @@ -40,8 +38,13 @@ services: environment: SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" SECRET_KEY: notsecret - #FLASK_ENV: production + #FLASK_ENV: production + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro command: > bash -c "flexmeasures db upgrade && flexmeasures add toy-account --name 'Docker Toy Account' && flexmeasures run --host 0.0.0.0 --port 5000" + +volumes: + flexmeasures-instance: \ No newline at end of file diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 1acf4521d..24182d1a9 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -14,18 +14,51 @@ We also support running all needed parts of a FlexMeasures EMS setup via `docker TODO: -- Ability to load config file into container (flexmeasures.cfg, if available) - Main Dockerfile serves API (via gunicorn and a WSGI file) +- Publish one flexmeasures image per version - Compose script defines one additional FM node, where it runs a worker as entry point instead. Document. - Some way to test that this is working, e.g. a list of steps. Document. Also include in Release list. Could be a test step, then a publish step. -- Fix: flask->flexmeasures (importlib error) -- Publish one flexmeasures image per version -Download and run the default image +The `flexmeasures` image ----------------------------------- -TODO +Building or downloading +^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can build the FlexMeasures image yourself: + + docker build -t flexmeasures/my-version . + +But you can also use versions we host at Docker Hub, e.g.: + + docker pull flexmeasures/flexmeasures:latest + + +Running +^^^^^^^^^^^ + +Running the image might work like this: + +.. code-block:: bash + + docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host your-image-name + +The two minimal environment variables are the database URI and the secret key. +In this example, we connect to a database running on our local computer, so we use the host net. +Browsing ``http://localhost:5000`` should work. + + +Configuring +^^^^^^^^^^^^^ + +Using the :ref:`configuration` by file is sometimes easier and also not all settings can be given via environment variables. +To load a configuration file into the container when starting up, you can put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: + +.. code-block:: bash + + docker run --volume flexmeasures-instance/:/var/usr/flexmeasures-instance/ -d --net=host your-image-name + Build the compose stack @@ -37,8 +70,6 @@ Run this: This builds the containers you need from code. If you change code, re-running this will re-build that image. -.. note:: Of course the ``pip install`` step takes time - maybe we want to try using pip install and caching: https://medium.com/@scythargon/cache-for-python-pip-downloads-and-wheels-in-docker-67f24e7cd84e) - Run the compose stack ------------------ @@ -65,8 +96,9 @@ To start a bash session in the `flexmeasures` container, do this: Configuration ---------------- -You can pass in your own configuration (e.g. MapBox access token, or db URI, see below) like this: TODO +You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: Put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``. +TODO: also load plugins this way (installing them by pip will be offered later) Data ----- From e303b8891670e4d42d697a98e54e3554a636520d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 12 Apr 2022 12:58:21 +0200 Subject: [PATCH 04/30] document better & correctly how to mount config file into a FlexMeasures container, clean up docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 82 ++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 24182d1a9..08d8de6d2 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -12,27 +12,26 @@ We also support running all needed parts of a FlexMeasures EMS setup via `docker .. warning:: The dockerization is still under development. -TODO: - -- Main Dockerfile serves API (via gunicorn and a WSGI file) -- Publish one flexmeasures image per version -- Compose script defines one additional FM node, where it runs a worker as entry point instead. Document. -- Some way to test that this is working, e.g. a list of steps. Document. Also include in Release list. Could be a test step, then a publish step. - The `flexmeasures` image ----------------------------------- -Building or downloading +Getting the image ^^^^^^^^^^^^^^^^^^^^^^^^^ -You can build the FlexMeasures image yourself: +You can use versions we host at Docker Hub, e.g.: - docker build -t flexmeasures/my-version . +.. code-block: bash + + docker pull flexmeasures:latest -But you can also use versions we host at Docker Hub, e.g.: - docker pull flexmeasures/flexmeasures:latest +You can also build the FlexMeasures image yourself, from source: + +.. code-block:: bash + + docker build -t flexmeasures/my-version . + Running @@ -45,64 +44,85 @@ Running the image might work like this: docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host your-image-name The two minimal environment variables are the database URI and the secret key. -In this example, we connect to a database running on our local computer, so we use the host net. + +In this example, we connect to a database running on our local computer, so we use the host network (in the docker-compose section below, we use a Docker container for the database, as well). + Browsing ``http://localhost:5000`` should work. -Configuring -^^^^^^^^^^^^^ +Configuration and customizing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables (a good example is the MapBox auth token, so you can load maps on the dashboard). -Using the :ref:`configuration` by file is sometimes easier and also not all settings can be given via environment variables. -To load a configuration file into the container when starting up, you can put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: +To load a configuration file into the container when starting up, you can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: .. code-block:: bash - docker run --volume flexmeasures-instance/:/var/usr/flexmeasures-instance/ -d --net=host your-image-name + docker run -v $(pwd)/flexmeasures-instance:/usr/var/flexmeasures-instance:ro -d --net=host flexmeasures/flexmeasures +.. note:: This is also a way to add your custom logic (as described in :ref:`plugins`) to the container. We'll document that shortly. Plugins which should be installed (e.g. by ``pip``) are a bit more difficult to support (you'd need to add `pip install` before the actual entry point). Ideas welcome. -Build the compose stack +The complete stack: compose -------------------------- +There are situations, for instance when developing or testing, when you want the whole stack of necessary nodes to be spun up by Docker. `Docker compose `_ is the answer for that. + + +Build the compose stack +^^^^^^^^^^^^^^^^^ + Run this: +.. code-block:: bash + docker-compose build -This builds the containers you need from code. If you change code, re-running this will re-build that image. +This pulls the containers you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. + +This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). + +.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). Run the compose stack ------------------- +^^^^^^^^^^^^^^^^^^^^^^ Start the stack like this: +.. code-block:: bash + docker-compose up You can see log output in the terminal, but ``docker-compose logs`` is also available to you. -Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running and ``docker-compose logs`` to look at output. ```docker inspect `` can be quite useful to dive into details. +Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running: -The FlexMeasures container has a health check implemented which is reflected in this output and you can see which ports are available on your machine to interact. +.. code-block:: console -Inspect individual containers -------------------------------- + ± docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 6105f6d1c91f flexmeasures_flexmeasures "bash -c 'flexmeasur…" 45 seconds ago Up 44 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_flexmeasures_1 + b48e4b9b113b postgres "docker-entrypoint.s…" 44 hours ago Up 45 seconds 5432/tcp flexmeasures_dev-db_1 -To start a bash session in the `flexmeasures` container, do this: - docker-compose run flexmeasures bash +The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact. + +You can use ``docker-compose logs`` to look at output. ``docker inspect `` and ``docker exec -it bash`` can be quite useful to dive into details. + +.. todo:: We should provide a way to test that this is working, e.g. a list of steps. Document this, but also include that in our tsc/Release list (as a test step to see if Dockerization still works, plus a publish step for the released version). Configuration ----------------- +^^^^^^^^^^^^^^ You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: Put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``. -TODO: also load plugins this way (installing them by pip will be offered later) Data ------ +^^^^^^ The postgres database is a test database with toy data filled in when the flexmeasures container starts. You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. - From b45e8057c9929f37da05e34103995b094cb16e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 12 Apr 2022 12:58:53 +0200 Subject: [PATCH 05/30] copy less unneeded files into FlexMeasures image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 3 ++- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2eac03095..724a68490 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,9 @@ RUN pip3 install --upgrade setuptools RUN pip3 install -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt # Copy code and meta/config data -COPY setup.* .flaskenv .env /app/ +COPY setup.* .flaskenv /app/ COPY flexmeasures/ /app/flexmeasures +RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf COPY .git/ /app/.git RUN pip3 install . diff --git a/docker-compose.yml b/docker-compose.yml index e15bc12ff..1f7ec4267 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: environment: SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" SECRET_KEY: notsecret - #FLASK_ENV: production + FLASK_ENV: development volumes: - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro command: > From d452d9f887f484504464c5662d34da4914b0154b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 12 Apr 2022 23:14:16 +0200 Subject: [PATCH 06/30] run the FlexMeasures image via Gunicorn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 28 ++++++++++++++-------------- docker-compose.yml | 6 ++++-- documentation/dev/docker.rst | 6 ++++-- wsgi.py | 3 +++ 4 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 wsgi.py diff --git a/Dockerfile b/Dockerfile index 724a68490..4ca7b8d6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN pip3 install --upgrade setuptools RUN pip3 install -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt # Copy code and meta/config data -COPY setup.* .flaskenv /app/ +COPY setup.* .flaskenv wsgi.py /app/ COPY flexmeasures/ /app/flexmeasures RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf COPY .git/ /app/.git @@ -28,16 +28,16 @@ COPY .git/ /app/.git RUN pip3 install . EXPOSE 5000 - -CMD ["flexmeasures", "run", "--host", "0.0.0.0", "--port", "5000"] - -#CMD [ \ -# "gunicorn", flexmeasures.wsgi", \ -# "--bind", "0.0.0.0:5000", \ -# # This is set to /tmp by default, but this is part of the Docker overlay filesystem, and can cause stalls. -# # http://docs.gunicorn.org/en/latest/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod -# "--worker-tmp-dir", "/dev/shm", \ -# # Ideally you'd want one worker per container, but we don't want to risk the health check timing out because -# # another request is taking a long time to complete. -# "--workers", "2", "--threads", "4" \ -#] +RUN apt-get install -y gunicorn + +CMD [ \ + "gunicorn", \ + "--bind", "0.0.0.0:5000", \ + # This is set to /tmp by default, but this is part of the Docker overlay filesystem, and can cause stalls. + # http://docs.gunicorn.org/en/latest/faq.html#how-do-i-avoid-gunicorn-excessively-blocking-in-os-fchmod + "--worker-tmp-dir", "/dev/shm", \ + # Ideally you'd want one worker per container, but we don't want to risk the health check timing out because + # another request is taking a long time to complete. + "--workers", "2", "--threads", "4", \ + "wsgi:application" \ +] diff --git a/docker-compose.yml b/docker-compose.yml index 1f7ec4267..e99965064 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ version: "3.8" services: dev-db: # This toy db is used with SQLALCHEMY_DATABASE_URI=postgresql://fm-dev-db-user:fm-dev-db-pass@localhost:5433/fm-dev-db - # You can set a different SQLALCHEMY_DATABASE_URI in .env, to work on a different database! image: postgres expose: - 5432 @@ -18,6 +17,7 @@ services: POSTGRES_DB: fm-dev-db POSTGRES_USER: fm-dev-db-user POSTGRES_PASSWORD: fm-dev-db-pass + #You could load & execute custom SQL like this: #volumes: # - ./fm-dev.sql:/docker-entrypoint-initdb.d/fm-dev.sql flexmeasures: @@ -40,11 +40,13 @@ services: SECRET_KEY: notsecret FLASK_ENV: development volumes: + # the first one is for running the FlexMeasures CLI, the 2nd for gunicorn - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro + - ./flexmeasures-instance/:/app/instance/:ro command: > bash -c "flexmeasures db upgrade && flexmeasures add toy-account --name 'Docker Toy Account' - && flexmeasures run --host 0.0.0.0 --port 5000" + && gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 2 --threads 4 wsgi:application" volumes: flexmeasures-instance: \ No newline at end of file diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 08d8de6d2..fd6e743c2 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -55,11 +55,13 @@ Configuration and customizing Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables (a good example is the MapBox auth token, so you can load maps on the dashboard). -To load a configuration file into the container when starting up, you can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: +To load a configuration file into the container when starting up, we make use of the `instance folder `_. You can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: .. code-block:: bash - docker run -v $(pwd)/flexmeasures-instance:/usr/var/flexmeasures-instance:ro -d --net=host flexmeasures/flexmeasures + docker run -v $(pwd)/flexmeasures-instance:/app/instance:ro -d --net=host flexmeasures/flexmeasures + +.. warning:: The location of the instance folder depends on how we serve FlexMeasures. The above works with gunicorn. See the compose file for an alternative (for the FlexMeasures CLI), and you can also read the above link about the instance folder. .. note:: This is also a way to add your custom logic (as described in :ref:`plugins`) to the container. We'll document that shortly. Plugins which should be installed (e.g. by ``pip``) are a bit more difficult to support (you'd need to add `pip install` before the actual entry point). Ideas welcome. diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 000000000..4da29c483 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,3 @@ +from flexmeasures.app import create as create_app + +application = create_app() From 382455fdfdb7947fe179ca19f258936753cf060f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 Apr 2022 15:46:07 +0200 Subject: [PATCH 07/30] install gunicorn early on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 3 +-- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ca7b8d6d..f9be4129f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 # pre-requisites -RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl +RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl gunicorn WORKDIR /app # requirements - doing this earlier, so we don't install them each time. Use --no-cache to refresh them. @@ -28,7 +28,6 @@ COPY .git/ /app/.git RUN pip3 install . EXPOSE 5000 -RUN apt-get install -y gunicorn CMD [ \ "gunicorn", \ diff --git a/docker-compose.yml b/docker-compose.yml index e99965064..bd2fed145 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,7 @@ services: SECRET_KEY: notsecret FLASK_ENV: development volumes: - # the first one is for running the FlexMeasures CLI, the 2nd for gunicorn + # a place for config and plugin code - the mount point is for running the FlexMeasures CLI, the 2nd for gunicorn - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro - ./flexmeasures-instance/:/app/instance/:ro command: > From 23f4a0e1bee2216c9dbf2d09113276daee5571f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 Apr 2022 18:06:37 +0200 Subject: [PATCH 08/30] add changelog entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 1b6f664af..36670c3f1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,12 +9,16 @@ v0.10.0 | April XX, 2022 New features ----------- +* API provides health check at /api/v3_0/health/ready [see `PR #416 `_] + Bugfixes ----------- Infrastructure / Support ---------------------- +* Dockerfile to run FlexMeasures API in container; also docker-compose file [see `PR #416 `_] + v0.9.1 | March 31, 2022 =========================== From 1689350f6ae014943a534f4db33709019b4969d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 Apr 2022 23:52:55 +0200 Subject: [PATCH 09/30] mention the Docker image on index and in toy tutorial section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 11 ++++++----- documentation/index.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 8 ++++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index fd6e743c2..ff1767a7c 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -3,7 +3,7 @@ Running via Docker ====================== -FlexMeasures can be run via `docker `_. TODO: link to the image once it's up. +FlexMeasures can be run via `docker `_. Docker is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. For now, the use case is local development. Using in production is a goal for later. @@ -21,9 +21,9 @@ Getting the image You can use versions we host at Docker Hub, e.g.: -.. code-block: bash +.. code-block:: bash - docker pull flexmeasures:latest + docker pull flexmeasures/flexmeasures:latest You can also build the FlexMeasures image yourself, from source: @@ -32,6 +32,7 @@ You can also build the FlexMeasures image yourself, from source: docker build -t flexmeasures/my-version . +The tag is your choice. Running @@ -41,7 +42,7 @@ Running the image might work like this: .. code-block:: bash - docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host your-image-name + docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host flexmeasures/flexmeasures The two minimal environment variables are the database URI and the secret key. @@ -83,7 +84,7 @@ Run this: This pulls the containers you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. -This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). +This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image form DockerHub. .. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). diff --git a/documentation/index.rst b/documentation/index.rst index bdcee6318..a998a1fa5 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -21,7 +21,7 @@ A tiny, but complete example: Let's install FlexMeasures from scratch. Then, usi .. code-block:: console - $ pip install flexmeasures + $ pip install flexmeasures # also available via Docker $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" && export SECRET_KEY=notsecret $ flexmeasures db upgrade # create tables diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 22b24c743..0beb64642 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -10,6 +10,8 @@ Let's walk through an example from scratch! We'll ... - load hourly prices - optimize a 12h-schedule for a battery that is half full +You'll need: A Unix computer with Python 3.8+, pip and `Docker `_. + Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat: .. code-block:: console @@ -30,9 +32,11 @@ Okay, let's get started! Install Flexmeasures and the database --------------------------------------- -This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python and two well-known developer tools, `pip `_ and `docker `_ +This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `docker `_. + +We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. -We install the FlexMeasures platform, use Docker to run a postgres database and tell FlexMeasures to create all tables. +.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker`). We plan to offer this tutorial for users, who only use Docker and run it completely inside containers. .. code-block:: console From 9ca44ef2899b37edc3bf4dac20b69f931c59e043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 13 Apr 2022 23:55:37 +0200 Subject: [PATCH 10/30] move the note about using Docker for the tutorial a little lower MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 0beb64642..b9d82eaea 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -36,8 +36,6 @@ This example is from scratch, so we'll assume you have nothing prepared but a (U We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. -.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker`). We plan to offer this tutorial for users, who only use Docker and run it completely inside containers. - .. code-block:: console $ pip install flexmeasures @@ -45,6 +43,8 @@ We start by installing the FlexMeasures platform, and then use Docker to run a p $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="WARNING" DEBUG=0 $ flexmeasures db upgrade +.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker`). We plan to offer this tutorial for users, who only use Docker and run it completely inside containers. + Add some structural data --------------------------------------- From fa71d31fedfca23b6b74e4b448481441407569cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 15 Apr 2022 00:03:18 +0200 Subject: [PATCH 11/30] install solver as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f9be4129f..7999d4731 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 # pre-requisites -RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl gunicorn +RUN apt-get update && apt-get install -y --upgrade python3 python3-pip git curl gunicorn coinor-cbc WORKDIR /app # requirements - doing this earlier, so we don't install them each time. Use --no-cache to refresh them. From 6e6c1e266a2075c790cd7e98b7f7d159932dfa21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 15 Apr 2022 00:04:10 +0200 Subject: [PATCH 12/30] document how at this point one can run tests inside a FlexMeasures container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index ff1767a7c..784977960 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -129,3 +129,14 @@ Data The postgres database is a test database with toy data filled in when the flexmeasures container starts. You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. + + +Running tests +^^^^^^^^^^^^^^ + +You can run tests in the flexmeasures docker container. This can be supported in a more straightforward way soon, of course, but here is how: + +- Go into the container: ``docker exec -it bash`` +- Install vim (or the editor of your choice): ``apt-get install vim`` +- Change the `SQLALCHEMY_DATABASE_URI` setting in ``flexmeasures/utils/config_defaults.py``, under "TestingConfig", to that in ``docker-compose.yml``. +- Run ``pytest``. \ No newline at end of file From cdb42e6b792037dcece6f6e19db33384dd2a9255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 15 Apr 2022 11:36:14 +0200 Subject: [PATCH 13/30] load sql extensions into postgres service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- ci/SETUP.sh | 2 +- ci/load-psql-extensions.sql | 2 ++ docker-compose.yml | 8 +++----- 3 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 ci/load-psql-extensions.sql diff --git a/ci/SETUP.sh b/ci/SETUP.sh index d2172fab7..836aa5cb6 100755 --- a/ci/SETUP.sh +++ b/ci/SETUP.sh @@ -30,4 +30,4 @@ while [[ true ]]; do fi done -psql -h $PGHOST -p $PGPORT -c "create extension if not exists cube; create extension if not exists earthdistance;" -U $PGUSER $PGDB; +psql -h $PGHOST -p $PGPORT --file ci/load-psql-extensions.sql -U $PGUSER $PGDB; diff --git a/ci/load-psql-extensions.sql b/ci/load-psql-extensions.sql new file mode 100644 index 000000000..3e1526617 --- /dev/null +++ b/ci/load-psql-extensions.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS cube; +CREATE EXTENSION IF NOT EXISTS earthdistance; diff --git a/docker-compose.yml b/docker-compose.yml index bd2fed145..c5bacaef6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,12 +3,11 @@ version: "3.8" # ----------------------------------------------- # TODO: # * Add redis service -# * Add a worker service, maybe using https://docs.docker.com/compose/compose-file/#entrypoint +# * Add a FlexMeasures worker service, maybe using https://docs.docker.com/compose/compose-file/#entrypoint # ----------------------------------------------- services: dev-db: - # This toy db is used with SQLALCHEMY_DATABASE_URI=postgresql://fm-dev-db-user:fm-dev-db-pass@localhost:5433/fm-dev-db image: postgres expose: - 5432 @@ -17,9 +16,8 @@ services: POSTGRES_DB: fm-dev-db POSTGRES_USER: fm-dev-db-user POSTGRES_PASSWORD: fm-dev-db-pass - #You could load & execute custom SQL like this: - #volumes: - # - ./fm-dev.sql:/docker-entrypoint-initdb.d/fm-dev.sql + volumes: + - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql flexmeasures: build: context: . From 1fd6b688d41a32d831fff3b2361ecc28b4249adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 15 Apr 2022 18:33:38 +0200 Subject: [PATCH 14/30] add another note to the preliminary testing setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 784977960..886bf401c 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -139,4 +139,6 @@ You can run tests in the flexmeasures docker container. This can be supported in - Go into the container: ``docker exec -it bash`` - Install vim (or the editor of your choice): ``apt-get install vim`` - Change the `SQLALCHEMY_DATABASE_URI` setting in ``flexmeasures/utils/config_defaults.py``, under "TestingConfig", to that in ``docker-compose.yml``. -- Run ``pytest``. \ No newline at end of file +- Run ``pytest``. (you might have to run it twice, as the database has to be wiped, which only happens afterwards) + +.. warning:: This will destroy data in the container. We probably will add a container to the compose stack just for testing. \ No newline at end of file From 101224ad22e1a5ef131604c7bb23386ba1eb59ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 15 Apr 2022 21:43:13 +0200 Subject: [PATCH 15/30] Run tests easily in Docker: add test-db service in compose stack, and a way to tell FlexMeasures to use a different database URI than the standard one when testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docker-compose.yml | 12 ++++++++++++ documentation/configuration.rst | 12 +++++++++++- documentation/dev/docker.rst | 13 +++++++------ flexmeasures/utils/config_utils.py | 6 +++++- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c5bacaef6..3a70ed5a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: - 5000:5000 depends_on: - dev-db + - test-db # use -e SQLALCHEMY_TEST_DATABASE_URI=... to exec pytest restart: on-failure healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5000/api/v3_0/health/ready"] @@ -45,6 +46,17 @@ services: bash -c "flexmeasures db upgrade && flexmeasures add toy-account --name 'Docker Toy Account' && gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 2 --threads 4 wsgi:application" + test-db: + image: postgres + expose: + - 5432 + restart: always + environment: + POSTGRES_DB: fm-test-db + POSTGRES_USER: fm-test-db-user + POSTGRES_PASSWORD: fm-test-db-pass + volumes: + - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql volumes: flexmeasures-instance: \ No newline at end of file diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 32c3129cd..c8bda48d5 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -6,7 +6,7 @@ Configuration The following configurations are used by FlexMeasures. Required settings (e.g. postgres db) are marked with a double star (**). -To enable easier quickstart tutorials, these settings can be set by environment variables. +To enable easier quickstart tutorials, these required settings can be set by environment variables. Recommended settings (e.g. mail, redis) are marked by one star (*). .. note:: FlexMeasures is best configured via a config file. The config file for FlexMeasures can be placed in one of two locations: @@ -302,6 +302,16 @@ Default: } +SQLALCHEMY_TEST_DATABASE_URI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When running tests (``make test``, which runs ``pytest``), the default database URI is set in ``utils.config_defaults.TestingConfig``. +You can use this setting to overwrite that URI and point the tests to an (empty) database of your choice. + +.. note:: This setting is only supported as an environment variable, not in a config file, and only during testing. + + + Security -------- diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 886bf401c..6ac185f46 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -134,11 +134,12 @@ You could also connect it to some other database, by setting a different `SQLALC Running tests ^^^^^^^^^^^^^^ -You can run tests in the flexmeasures docker container. This can be supported in a more straightforward way soon, of course, but here is how: +You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file. Per default, we are using the ``dev-db`` database service. -- Go into the container: ``docker exec -it bash`` -- Install vim (or the editor of your choice): ``apt-get install vim`` -- Change the `SQLALCHEMY_DATABASE_URI` setting in ``flexmeasures/utils/config_defaults.py``, under "TestingConfig", to that in ``docker-compose.yml``. -- Run ``pytest``. (you might have to run it twice, as the database has to be wiped, which only happens afterwards) +After you've started the compose stack with ``docker-compose up``, run: -.. warning:: This will destroy data in the container. We probably will add a container to the compose stack just for testing. \ No newline at end of file +.. code-block:: console + + docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" pytest + +This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. \ No newline at end of file diff --git a/flexmeasures/utils/config_utils.py b/flexmeasures/utils/config_utils.py index 4a7d28acb..a786b27ad 100644 --- a/flexmeasures/utils/config_utils.py +++ b/flexmeasures/utils/config_utils.py @@ -78,12 +78,16 @@ def read_config(app: Flask, custom_path_to_config: Optional[str]): path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") - # Don't overwrite when testing (that should run completely on defaults) + # Custom config: not when testing (that should run completely on defaults) if not app.testing: used_path_to_config = read_custom_config( app, custom_path_to_config, path_to_config_home, path_to_config_instance ) read_env_vars(app) + else: # one exception: the ability to set where the test database is + custom_test_db_uri = os.getenv("SQLALCHEMY_TEST_DATABASE_URI", None) + if custom_test_db_uri: + app.config["SQLALCHEMY_DATABASE_URI"] = custom_test_db_uri # Check for missing values. # Documentation runs fine without them. From 9adc8b76c68ca03a37a23e8075c729a7fd862d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 16 Apr 2022 22:20:12 +0200 Subject: [PATCH 16/30] improve documentation, from review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 5 ++-- documentation/dev/docker.rst | 28 +++++++++++-------- .../tut/toy-example-from-scratch.rst | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4789a9606..63b1d518b 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,14 +9,14 @@ v0.10.0 | April XX, 2022 New features ----------- * Improve legibility of chart axes [see `PR #413 `_] - -* API provides health check at /api/v3_0/health/ready [see `PR #416 `_] +* API provides health readiness check at /api/v3_0/health/ready [see `PR #416 `_] Bugfixes ----------- Infrastructure / Support ---------------------- +* Dockerfile to run FlexMeasures API in container; also docker-compose file [see `PR #416 `_] * Unit conversion prefers shorter units in general [see `PR #415 `_] * Shorter CI builds in Github Actions by caching Python environment [see `PR #361 `_] @@ -37,7 +37,6 @@ Bugfixes * Prefer unit conversions to short stock units [see `PR #412 `_] * Fix filter for selecting one deterministic belief per event, which was duplicating index levels [see `PR #414 `_] -* Dockerfile to run FlexMeasures API in container; also docker-compose file [see `PR #416 `_] v0.9.1 | March 31, 2022 diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 6ac185f46..874a7b98c 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -5,12 +5,12 @@ Running via Docker FlexMeasures can be run via `docker `_. -Docker is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. +`Docker `_ is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. For now, the use case is local development. Using in production is a goal for later. We also support running all needed parts of a FlexMeasures EMS setup via `docker-compose `_, which is helpful for developers and might inform hosting efforts. -.. warning:: The dockerization is still under development. +.. warning:: The dockerization is still `under development `_. The `flexmeasures` image @@ -38,23 +38,27 @@ The tag is your choice. Running ^^^^^^^^^^^ -Running the image might work like this: +Running the image (as a container) might work like this (remember to get the image first, see above): .. code-block:: bash docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host flexmeasures/flexmeasures -The two minimal environment variables are the database URI and the secret key. +.. note:: Don't know what your image is called (it's "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. -In this example, we connect to a database running on our local computer, so we use the host network (in the docker-compose section below, we use a Docker container for the database, as well). +The two minimal environment variables to run the container successfully are the database URI and the secret key. -Browsing ``http://localhost:5000`` should work. +In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well. +Browsing ``http://localhost:5000`` should work now and ask you to log in. -Configuration and customizing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Of course, you might not have created a user. You can use ``docker exec -it bash`` to go inside the container and use the :ref:`cli` to create everything you need. -Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables (a good example is the MapBox auth token, so you can load maps on the dashboard). + +Configuration and customization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :ref:`configuration` by file is usually what you want to do. It's easier than adding environment variables to ``docker run``. Also, not all settings can be given via environment variables. A good example is the :ref:`mapbox_access_token`, so you can load maps on the dashboard. To load a configuration file into the container when starting up, we make use of the `instance folder `_. You can put a configuration file called ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance`` and then mount that folder into the container, like this: @@ -82,9 +86,9 @@ Run this: docker-compose build -This pulls the containers you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. +This pulls the images you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. -This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image form DockerHub. +This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub. .. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). @@ -121,7 +125,7 @@ You can use ``docker-compose logs`` to look at output. ``docker inspect Date: Sun, 17 Apr 2022 13:11:51 +0200 Subject: [PATCH 17/30] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 874a7b98c..7b64309e1 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -44,7 +44,7 @@ Running the image (as a container) might work like this (remember to get the ima docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host flexmeasures/flexmeasures -.. note:: Don't know what your image is called (it's "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. +.. note:: Don't know what your image is called (its "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. The two minimal environment variables to run the container successfully are the database URI and the secret key. From 79a99c05bc25fee184453641162722b2dfc1b35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 17 Apr 2022 13:23:19 +0200 Subject: [PATCH 18/30] lazy loading in SensorIdField (fixes flexmeasures add schedule) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/schemas/sensors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 6f96cbcae..4a06f306d 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -65,6 +65,8 @@ def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: sensor = Sensor.query.get(value) if sensor is None: raise FMValidationError(f"No sensor found with id {value}.") + # lazy loading now (sensor is somehow not in session after this) + sensor.generic_asset return sensor def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: From 8bdf8115fc0e0914fb8f07a2fe6f1397d045e1f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 17 Apr 2022 14:08:48 +0200 Subject: [PATCH 19/30] docs: add FLASK_ENV=development to docker run call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/dev/docker.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/dev/docker.rst b/documentation/dev/docker.rst index 7b64309e1..97c37e2eb 100644 --- a/documentation/dev/docker.rst +++ b/documentation/dev/docker.rst @@ -42,11 +42,11 @@ Running the image (as a container) might work like this (remember to get the ima .. code-block:: bash - docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla -d --net=host flexmeasures/flexmeasures + docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla --env FLASK_ENV=development -d --net=host flexmeasures/flexmeasures .. note:: Don't know what your image is called (its "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. -The two minimal environment variables to run the container successfully are the database URI and the secret key. +The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``Flask_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well. From 5c65024bef58949cce75bcdec9bd311b0d0ec027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 18 Apr 2022 17:34:20 +0200 Subject: [PATCH 20/30] split docker and docker-compose docs (first is relevant for installing/deploying), the latter more for development. Also fix small errors and give better info on WSGI scripts. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- .gitignore | 1 + Makefile | 2 +- documentation/conf.py | 4 +- documentation/dev/ci.rst | 14 +++- documentation/dev/docker-compose.rst | 80 ++++++++++++++++++ documentation/host/deployment.rst | 18 +++-- documentation/{dev => host}/docker.rst | 81 +------------------ documentation/index.rst | 5 +- documentation/tut/installation.rst | 14 ++-- .../tut/toy-example-from-scratch.rst | 2 +- 10 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 documentation/dev/docker-compose.rst rename documentation/{dev => host}/docker.rst (51%) diff --git a/.gitignore b/.gitignore index 519e917df..a5084d6b7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ venv .env-flexmeasures/ flexmeasures.egg-info flexmeasures.log* +prices-tomorrow.csv .cache __pycache__ diff --git a/Makefile b/Makefile index 7a1436707..85ae39362 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ update-docs: @echo "Creating docs environment ..." make install-docs-dependencies @echo "Creating documentation ..." - cd documentation; make clean; make html; cd .. + cd documentation; make clean; make html SPHINXOPTS="-W --keep-going -n"; cd .. update-docs-pdf: @echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)" diff --git a/documentation/conf.py b/documentation/conf.py index 763a72084..0b0b19faa 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -77,7 +77,7 @@ # This pattern also affects html_static_path and html_extra_path . exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] -# Todo: these are not mature enough yet for release +# Todo: these are not mature enough yet for release, or should be removed exclude_patterns.append("int/*.rst") exclude_patterns.append("concepts/assets.rst") exclude_patterns.append("concepts/markets.rst") @@ -194,7 +194,7 @@ # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} +intersphinx_mapping = {"https://docs.python.org/3/": None} # -- Options for copybytton extension --------------------------------------- copybutton_prompt_is_regexp = True diff --git a/documentation/dev/ci.rst b/documentation/dev/ci.rst index 715e27aca..222f27a9a 100644 --- a/documentation/dev/ci.rst +++ b/documentation/dev/ci.rst @@ -1,4 +1,4 @@ -.. continuous_integration: +.. _continuous_integration: Continuous integration ====================== @@ -97,3 +97,15 @@ The last step, touching a wsgi.py file, is often used as a way to soft-restart t echo "RESTARTING APPLICATION ..." touch $PATH_TO_WSGI + + +A WSGI file can do various things, as well, but the simplest form is shown below. + +.. code-block:: python + + from flexmeasures.app import create as create_app + + application = create_app() + + +The web server is told about the WSGI script, but also about the object which represents the application. For instance, if this script is called ``wsgi.py``, then the relevant argument to the gunicorn server is ``wsgi:application``. \ No newline at end of file diff --git a/documentation/dev/docker-compose.rst b/documentation/dev/docker-compose.rst new file mode 100644 index 000000000..1bbc8862a --- /dev/null +++ b/documentation/dev/docker-compose.rst @@ -0,0 +1,80 @@ +.. _docker-compose: + +Running a complete stack with docker-compose +============================================= + +Installing FlexMeasures and dependencies is some work, and can have unexpected hurdles, e.g. depending on the operating system. So, running the whole stack via `Docker compose `_ can be a very useful option. + +For this, we assume you are in the directory housing ``docker-compose.yml``. + + +Build the compose stack +------------------------ + +Run this: + +.. code-block:: bash + + docker-compose build + +This pulls the images you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. + +This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub. + +.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). + + +Run the compose stack +---------------------- + +Start the stack like this: + +.. code-block:: bash + + docker-compose up + +You can see log output in the terminal, but ``docker-compose logs`` is also available to you. + +Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running: + + +.. code-block:: console + + ± docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 6105f6d1c91f flexmeasures_flexmeasures "bash -c 'flexmeasur…" 45 seconds ago Up 44 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_flexmeasures_1 + b48e4b9b113b postgres "docker-entrypoint.s…" 44 hours ago Up 45 seconds 5432/tcp flexmeasures_dev-db_1 + + +The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact. + +You can use ``docker-compose logs`` to look at output. ``docker inspect `` and ``docker exec -it bash`` can be quite useful to dive into details. + +.. todo:: We should provide a way to test that this is working, e.g. a list of steps. Document this, but also include that in our tsc/Release list (as a test step to see if Dockerization still works, plus a publish step for the released version). + + +Configuration +--------------- + +You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``. + + +Data +------- + +The postgres database is a test database with toy data filled in when the flexmeasures container starts. +You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. + + +Running tests +--------------- + +You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file. Per default, we are using the ``dev-db`` database service. + +After you've started the compose stack with ``docker-compose up``, run: + +.. code-block:: console + + docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" pytest + +This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. \ No newline at end of file diff --git a/documentation/host/deployment.rst b/documentation/host/deployment.rst index 55c4e6a03..4306bae16 100644 --- a/documentation/host/deployment.rst +++ b/documentation/host/deployment.rst @@ -5,7 +5,7 @@ How to deploy FlexMeasures Here you can learn how to get FlexMeasures onto a server. -.. note:: FlexMeasures can be deployed via Docker. Read more at :ref:`docker`. +.. note:: FlexMeasures can be deployed via Docker. Read more at :ref:`docker-image`. .. contents:: Table of contents :local: @@ -23,17 +23,25 @@ On your own computer, ``flexmeasures run`` is a nice way to start FlexMeasures. # This file contains the WSGI configuration required to serve up your # web application. # It works by setting the variable 'application' to a WSGI handler of some description. + # The crucial part are the last two lines. We add some ideas for possible other logic. - # use this if you serve from code import os - from dotenv import load_dotenv project_home = u'/path/to/your/code/flexmeasures' + # use this if you want to load your own ``.env`` file. + from dotenv import load_dotenv load_dotenv(os.path.join(project_home, '.env')) + # use this if you run from source + if project_home not in sys.path: + sys.path = [project_home] + sys.path + # adapt PATH to find our LP solver if it is installed from source + os.environ["PATH"] = os.environ.get("PATH") + ":/home/seita/Cbc-2.9/bin" - # create flask app - need to call it "application" for WSGI to work + # create flask app - the name "application" has to be passed to the WSGI server from flexmeasures.app import create as create_app application = create_app() +The web server is told about the WSGI script, but also about the object which represents the application. For instance, if this script is called ``wsgi.py``, then the relevant argument to the gunicorn server is ``wsgi:application``. + Keep in mind that FlexMeasures is based on `Flask `_, so almost all knowledge on the web on how to deploy a Flask app also helps with deploying FlexMeasures. @@ -53,7 +61,7 @@ You can install it on Debian like this: If you can't use the package manager on your host, the solver has to be installed from source. -We provide `an example script `_ to do that, where you can also +We provide an example script in ``ci/install-cbc-from-source.sh`` to do that, where you can also pass a directory for the installation. In case you want to install a later version, adapt the version in the script. diff --git a/documentation/dev/docker.rst b/documentation/host/docker.rst similarity index 51% rename from documentation/dev/docker.rst rename to documentation/host/docker.rst index 97c37e2eb..d4a0b568c 100644 --- a/documentation/dev/docker.rst +++ b/documentation/host/docker.rst @@ -1,4 +1,4 @@ -.. docker: +.. _docker-image: Running via Docker ====================== @@ -12,6 +12,8 @@ We also support running all needed parts of a FlexMeasures EMS setup via `docker .. warning:: The dockerization is still `under development `_. +We also provide a docker-compose file for development, see :ref:`docker-compose`. + The `flexmeasures` image ----------------------------------- @@ -70,80 +72,3 @@ To load a configuration file into the container when starting up, we make use of .. note:: This is also a way to add your custom logic (as described in :ref:`plugins`) to the container. We'll document that shortly. Plugins which should be installed (e.g. by ``pip``) are a bit more difficult to support (you'd need to add `pip install` before the actual entry point). Ideas welcome. - -The complete stack: compose --------------------------- - -There are situations, for instance when developing or testing, when you want the whole stack of necessary nodes to be spun up by Docker. `Docker compose `_ is the answer for that. - - -Build the compose stack -^^^^^^^^^^^^^^^^^ - -Run this: - -.. code-block:: bash - - docker-compose build - -This pulls the images you need, and re-builds the FlexMeasures one from code. If you change code, re-running this will re-build that image. - -This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub. - -.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). - - -Run the compose stack -^^^^^^^^^^^^^^^^^^^^^^ - -Start the stack like this: - -.. code-block:: bash - - docker-compose up - -You can see log output in the terminal, but ``docker-compose logs`` is also available to you. - -Check ``docker ps`` or ``docker-compose ps`` to see if your containers are running: - - -.. code-block:: console - - ± docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 6105f6d1c91f flexmeasures_flexmeasures "bash -c 'flexmeasur…" 45 seconds ago Up 44 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_flexmeasures_1 - b48e4b9b113b postgres "docker-entrypoint.s…" 44 hours ago Up 45 seconds 5432/tcp flexmeasures_dev-db_1 - - -The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact. - -You can use ``docker-compose logs`` to look at output. ``docker inspect `` and ``docker exec -it bash`` can be quite useful to dive into details. - -.. todo:: We should provide a way to test that this is working, e.g. a list of steps. Document this, but also include that in our tsc/Release list (as a test step to see if Dockerization still works, plus a publish step for the released version). - - -Configuration -^^^^^^^^^^^^^^ - -You can pass in your own configuration (e.g. for MapBox access token, or db URI, see below) like we described above for running a container: put a file ``flexmeasures.cfg`` into a local folder called ``flexmeasures-instance``. - - -Data -^^^^^^ - -The postgres database is a test database with toy data filled in when the flexmeasures container starts. -You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. - - -Running tests -^^^^^^^^^^^^^^ - -You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file. Per default, we are using the ``dev-db`` database service. - -After you've started the compose stack with ``docker-compose up``, run: - -.. code-block:: console - - docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" pytest - -This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. \ No newline at end of file diff --git a/documentation/index.rst b/documentation/index.rst index 7a065268c..75fc9eef1 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -165,8 +165,9 @@ The platform operator of FlexMeasures can be an Aggregator. :caption: Hosting FlexMeasures :maxdepth: 1 - host/deployment + host/docker host/data + host/deployment host/error-monitoring host/modes @@ -189,7 +190,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/api dev/ci dev/auth - dev/docker + dev/docker-compose diff --git a/documentation/tut/installation.rst b/documentation/tut/installation.rst index 0888a0ccd..0d5f94ec2 100644 --- a/documentation/tut/installation.rst +++ b/documentation/tut/installation.rst @@ -6,9 +6,14 @@ Installation & First steps Getting FlexMeasures to run ----------------------------- -This section walks you through getting FlexMeasures to run. For an example with the least effort, see :ref:`tut_toy_schedule`. Here, we'll cover getting started with an installation you run continuously ― making a secret key, connecting a database and creating one user & one asset. +This section walks you through installing FlexMeasures on your own PC and running it continuously. +We'll cover getting started by making a secret key, connecting a database and creating one user & one asset. -.. note:: Are you not hosting FlexMeasures, but want to learn how to interact with it? Start with :ref:`tut_posting_data`. +.. note:: Maybe these starting points are also interesting for you: + + * For an example to see FlexMeasures in action with the least effort, see :ref:`tut_toy_schedule`. + * You can run FlexMeasures via Docker, see :ref:`docker` and :ref:`docker-compose`. + * Are you not hosting FlexMeasures, but want to learn how to interact with it? Start with :ref:`tut_posting_data`. Install FlexMeasures @@ -205,14 +210,13 @@ It's finally time to start running FlexMeasures: (This might print some warnings, see the next section where we go into more detail) .. note:: In a production context, you shouldn't run a script - hand the ``app`` object to a WSGI process, as your platform of choice describes. - Often, that requires a WSGI script. We provide an example WSGI script in :ref:`continuous_integration`. + Often, that requires a WSGI script. We provide an example WSGI script in :ref:`continuous_integration`. You can also take a look at FlexMeasures' Dockerfile to get an idea how to run FlexMeasures with gunicorn. You can visit ``http://localhost:5000`` now to see if the app's UI works. When you see the dashboard, the map will not work. For that, you'll need to get your :ref:`mapbox_access_token` and add it to your config file. - Other settings, for full functionality -------------------------------------- @@ -258,4 +262,4 @@ Where to go from here? If your data structure is good, you should think about (continually) adding measurement data. This tutorial mentioned how to add data, but :ref:`_tut_posting_data` goes deeper with examples and terms & definitions. -Then, you probably want to use FlexMeasures to generate forecasts and schedules! For this, read further in :ref:`_tut_forecasting_scheduling`. \ No newline at end of file +Then, you probably want to use FlexMeasures to generate forecasts and schedules! For this, read further in :ref:`tut_forecasting_scheduling`. \ No newline at end of file diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 51e7fea02..d073d83d4 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -43,7 +43,7 @@ We start by installing the FlexMeasures platform, and then use Docker to run a p $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="WARNING" DEBUG=0 $ flexmeasures db upgrade -.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker`). We plan to rewrite this tutorial to include steps for users running the complete FlexMeasures platform inside docker containers. +.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker-image`). We plan to rewrite this tutorial to include steps for users running the complete FlexMeasures platform inside docker containers. Add some structural data From 70a712d83672b4a2b328d36402d5a0ac684f82bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 19 Apr 2022 19:58:54 +0200 Subject: [PATCH 21/30] fix regression in not attaching account to assets correctly in toy tutorial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 828e38e06..dfaeab002 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -959,7 +959,7 @@ def add_toy_account(kind: str, name: str): asset = GenericAsset( name=f"toy-{asset_type}", generic_asset_type=asset_types[asset_type], - owner=account, + owner=user.account, latitude=location[0], longitude=location[1], ) From 822d374a40782eceac06c1f9fdaf988d819cd647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 20 Apr 2022 21:40:30 +0200 Subject: [PATCH 22/30] separate installation instructions for toy tutorial into Docker and non-Docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docker-compose.yml | 3 +- documentation/conf.py | 1 + .../tut/toy-example-from-scratch.rst | 65 ++++++++++++++++--- requirements/app.txt | 1 - requirements/docs.in | 1 + requirements/docs.txt | 8 ++- 6 files changed, 66 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a70ed5a2..272d06295 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,8 @@ services: environment: SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" SECRET_KEY: notsecret - FLASK_ENV: development + FLASK_ENV: development + LOGGING_LEVEL: INFO volumes: # a place for config and plugin code - the mount point is for running the FlexMeasures CLI, the 2nd for gunicorn - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:ro diff --git a/documentation/conf.py b/documentation/conf.py index 0b0b19faa..961df47c6 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -48,6 +48,7 @@ "sphinx.ext.ifconfig", "sphinx.ext.todo", "sphinx_copybutton", + "sphinx_tabs.tabs", "sphinx_fontawesome", "sphinxcontrib.autohttp.flask", "sphinxcontrib.autohttp.flaskqref", diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index d073d83d4..592fdebea 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -10,7 +10,7 @@ Let's walk through an example from scratch! We'll ... - load hourly prices - optimize a 12h-schedule for a battery that is half full -You'll need: A Unix computer with Python 3.8+, pip and `Docker `_. +You'll do this on your own computer, either with `Docker `_ or on a Unix computer with Python 3.8+, pip and postgres. Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat: @@ -29,21 +29,66 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s Okay, let's get started! +.. note: You can copy the commands by hovering on the top right corner of code examples. You'll copy only the commands, not the output! + Install Flexmeasures and the database --------------------------------------- -This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `docker `_. +.. tabs:: -We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. + .. tab:: Docker -.. code-block:: console + Okay, if `docker `_ is running on your system, you're good to go. Otherwise, see `here `_ + + We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. + + .. code-block:: console + + $ docker pull flexmeasures/flexmeasures:latest + $ docker pull postgres + $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest + $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:docker@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host flexmeasures/flexmeasures + $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade" + + Now the rest of this tutorial will happen inside the FlexMeasures container. This is how you hop inside the container and run a terminal there: + + .. code-block:: console + + $ docker exec -it flexmeasures-tutorial-fm bash + + To leave the container session, hold CTRL-C or type "exit". + + To stop the containers, you can type + + .. code-block:: console + + $ docker stop flexmeasures-tutorial-db + $ docker stop flexmeasures-tutorial-fm + + + .. tab:: On your PC + + This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `postgres > 9`_. + + + We'll create a databae for FlexMeasures: + + .. code-block:: console + + sudo -i -u postgres + createdb -U postgres flexmeasures + createuser --pwprompt -U postgres flexmeasures # enter your password, we'll use "flexmeasures" + exit + + Then, we can install FlexMeasures itself, set some variables and tell FlexMeasures to create all tables: + + .. code-block:: console - $ pip install flexmeasures - $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest - $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="WARNING" DEBUG=0 - $ flexmeasures db upgrade + $ pip install flexmeasures + $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures:flexmeasures@127.0.0.1:5432/flexmeasures" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0 + $ flexmeasures db upgrade -.. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use the FlexMeasures Docker image (see :ref:`docker-image`). We plan to rewrite this tutorial to include steps for users running the complete FlexMeasures platform inside docker containers. + .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which usesa prepared Linux image, so it'll definitely work. Add some structural data @@ -91,7 +136,7 @@ If you want, you can inspect what you created: 2 toy-building building (52.374, 4.88969) 1 toy-solar solar (52.374, 4.88969) - $ flexmeasures show asset --id + $ flexmeasures show asset --id 3 =========================== Asset toy-battery (ID:3): diff --git a/requirements/app.txt b/requirements/app.txt index 87534b8ee..08af442c9 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -219,7 +219,6 @@ packaging==21.3 # via # bokeh # matplotlib - # pint # redis # statsmodels # webargs diff --git a/requirements/docs.in b/requirements/docs.in index 96f695783..49cc0f542 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -5,3 +5,4 @@ sphinx-rtd-theme sphinxcontrib.httpdomain sphinx_fontawesome sphinx_copybutton +sphinx-tabs diff --git a/requirements/docs.txt b/requirements/docs.txt index 17ed18975..d830d8497 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -32,6 +32,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme + # sphinx-tabs idna==3.3 # via # -c requirements/app.txt @@ -60,7 +61,9 @@ pycparser==2.21 # -c requirements/app.txt # cffi pygments==2.11.2 - # via sphinx + # via + # sphinx + # sphinx-tabs pyopenssl==22.0.0 # via # -c requirements/app.txt @@ -89,6 +92,7 @@ sphinx==4.4.0 # sphinx-copybutton # sphinx-fontawesome # sphinx-rtd-theme + # sphinx-tabs # sphinxcontrib.httpdomain sphinx-copybutton==0.5.0 # via -r requirements/docs.in @@ -96,6 +100,8 @@ sphinx-fontawesome==0.0.6 # via -r requirements/docs.in sphinx-rtd-theme==1.0.0 # via -r requirements/docs.in +sphinx-tabs==3.3.1 + # via -r requirements/docs.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 From adbfe2527d65f1de8e6f5437ff0dea5ac848ad66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 20 Apr 2022 23:50:03 +0200 Subject: [PATCH 23/30] some text improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/tut/toy-example-from-scratch.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 592fdebea..f996b51f6 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -10,7 +10,7 @@ Let's walk through an example from scratch! We'll ... - load hourly prices - optimize a 12h-schedule for a battery that is half full -You'll do this on your own computer, either with `Docker `_ or on a Unix computer with Python 3.8+, pip and postgres. +What do you need? Your own computer, with one of two situations: Either you have `Docker `_ or your computer supports Python 3.8+, pip and PostgresDB. The former might be easier, see the installation step below. But you choose. Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain step by step. There are some other crucial steps for installation and setup, so this becomes a complete example from scratch, but this is the meat: @@ -29,7 +29,7 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s Okay, let's get started! -.. note: You can copy the commands by hovering on the top right corner of code examples. You'll copy only the commands, not the output! +.. note:: You can copy the commands by hovering on the top right corner of code examples. You'll copy only the commands, not the output! Install Flexmeasures and the database --------------------------------------- @@ -38,7 +38,7 @@ Install Flexmeasures and the database .. tab:: Docker - Okay, if `docker `_ is running on your system, you're good to go. Otherwise, see `here `_ + If `docker `_ is running on your system, you're good to go. Otherwise, see `here `_. We start by installing the FlexMeasures platform, and then use Docker to run a postgres database and tell FlexMeasures to create all tables. @@ -68,10 +68,9 @@ Install Flexmeasures and the database .. tab:: On your PC - This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `postgres > 9`_. + This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip `_ and `postgres `_. - - We'll create a databae for FlexMeasures: + We'll create a database for FlexMeasures: .. code-block:: console @@ -88,7 +87,7 @@ Install Flexmeasures and the database $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures:flexmeasures@127.0.0.1:5432/flexmeasures" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0 $ flexmeasures db upgrade - .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which usesa prepared Linux image, so it'll definitely work. + .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which uses a prepared Linux image, so it'll definitely work. Add some structural data From b695c0d42d83466552131f84d2349e44aebe1cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 21 Apr 2022 14:34:42 +0200 Subject: [PATCH 24/30] correct the link in coverage badge, to report on main branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index 514a6abb1..9513b212e 100644 --- a/Readme.md +++ b/Readme.md @@ -6,7 +6,7 @@ [![](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Documentation Status](https://readthedocs.org/projects/flexmeasures/badge/?version=latest)](https://flexmeasures.readthedocs.io/en/latest/?badge=latest) -[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg?branch=coverage-in-ci)](https://coveralls.io/github/FlexMeasures/flexmeasures?branch=coverage-in-ci) +[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg?branch=coverage-in-ci)](https://coveralls.io/github/FlexMeasures/flexmeasures) The *FlexMeasures Platform* is the intelligent EMS (energy management system) to support real-time energy flexibility apps, rapidly and scalable. @@ -42,4 +42,4 @@ We made FlexMeasures freely available under the Apache2.0 licence and it is now Within the FlexMeasures project, [we welcome contributions](https://github.com/FlexMeasures/tsc/blob/main/CONTRIBUTING.md). You can also [learn more about our governance](https://github.com/Flexmeasures/tsc/blob/main/GOVERNANCE.md). -You can connect with the community here on Github (e.g. by creating an issue), on [the mailing list](https://lists.lfenergy.org/g/flexmeasures), on [the FlexMeasures channel within the LF Energy Slack](https://slack.lfenergy.org/) or [by contacting the current maintainers](https://www.seita.nl/contact). \ No newline at end of file +You can connect with the community here on Github (e.g. by creating an issue), on [the mailing list](https://lists.lfenergy.org/g/flexmeasures), on [the FlexMeasures channel within the LF Energy Slack](https://slack.lfenergy.org/) or [by contacting the current maintainers](https://www.seita.nl/contact). From a2237df7b64d8f8d51ba651de69c6bd7a29b4a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 22 Apr 2022 09:08:39 +0200 Subject: [PATCH 25/30] switch to lfenergy/flexmeasures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/host/docker.rst | 10 +++++----- documentation/index.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/documentation/host/docker.rst b/documentation/host/docker.rst index d4a0b568c..7af737aba 100644 --- a/documentation/host/docker.rst +++ b/documentation/host/docker.rst @@ -25,7 +25,7 @@ You can use versions we host at Docker Hub, e.g.: .. code-block:: bash - docker pull flexmeasures/flexmeasures:latest + docker pull lfenergy/flexmeasures:latest You can also build the FlexMeasures image yourself, from source: @@ -44,11 +44,11 @@ Running the image (as a container) might work like this (remember to get the ima .. code-block:: bash - docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla --env FLASK_ENV=development -d --net=host flexmeasures/flexmeasures + docker run --env SQLALCHEMY_DATABASE_URI=postgresql://user:pass@localhost:5432/dbname --env SECRET_KEY=blabla --env FLASK_ENV=development -d --net=host lfenergy/flexmeasures -.. note:: Don't know what your image is called (its "tag")? We used ``flexmeasures/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. +.. note:: Don't know what your image is called (its "tag")? We used ``lfenergy/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. -The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``Flask_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). +The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``Flask_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). If you see too much output, you can also set ``LOGGING_LEVEL=INFO``. In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well. @@ -66,7 +66,7 @@ To load a configuration file into the container when starting up, we make use of .. code-block:: bash - docker run -v $(pwd)/flexmeasures-instance:/app/instance:ro -d --net=host flexmeasures/flexmeasures + docker run -v $(pwd)/flexmeasures-instance:/app/instance:ro -d --net=host lfenergy/flexmeasures .. warning:: The location of the instance folder depends on how we serve FlexMeasures. The above works with gunicorn. See the compose file for an alternative (for the FlexMeasures CLI), and you can also read the above link about the instance folder. diff --git a/documentation/index.rst b/documentation/index.rst index 75fc9eef1..8775b1baa 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -44,7 +44,7 @@ As possible users, we see energy service companies (ESCOs) who want to build rea However, even small companies and hobby projects might find FlexMeasures useful! We are constantly improving the ease of use. -FlexMeasures can be used as your EMS, but is also to integrate with existing systems as a smart backend or add-on to deal with energy flexibility specifically. +FlexMeasures can be used as your EMS, but it can also integrate with existing systems as a smart backend, or as an add-on to deal with energy flexibility specifically. The image below shows how FlexMeasures, with the help of plugins fitted for a given use case, turns data into optimized schedules: diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index f996b51f6..5c7ddd34b 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -44,10 +44,10 @@ Install Flexmeasures and the database .. code-block:: console - $ docker pull flexmeasures/flexmeasures:latest + $ docker pull lfenergy/flexmeasures:latest $ docker pull postgres $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest - $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:docker@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host flexmeasures/flexmeasures + $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:docker@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host lfenergy/flexmeasures $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade" Now the rest of this tutorial will happen inside the FlexMeasures container. This is how you hop inside the container and run a terminal there: From 89a98333a313f8fee06dd455bfca27de506a63a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 22 Apr 2022 21:41:33 +0200 Subject: [PATCH 26/30] small comments from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- .gitignore | 2 +- Readme.md | 2 +- documentation/dev/docker-compose.rst | 2 +- documentation/host/docker.rst | 4 ++-- documentation/tut/toy-example-from-scratch.rst | 10 +++++----- flexmeasures/cli/data_add.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index a5084d6b7..38ef57bce 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ venv .env-flexmeasures/ flexmeasures.egg-info flexmeasures.log* -prices-tomorrow.csv +*.csv .cache __pycache__ diff --git a/Readme.md b/Readme.md index 9513b212e..c6c0cd090 100644 --- a/Readme.md +++ b/Readme.md @@ -6,7 +6,7 @@ [![](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Documentation Status](https://readthedocs.org/projects/flexmeasures/badge/?version=latest)](https://flexmeasures.readthedocs.io/en/latest/?badge=latest) -[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg?branch=coverage-in-ci)](https://coveralls.io/github/FlexMeasures/flexmeasures) +[![Coverage](https://coveralls.io/repos/github/FlexMeasures/flexmeasures/badge.svg)](https://coveralls.io/github/FlexMeasures/flexmeasures) The *FlexMeasures Platform* is the intelligent EMS (energy management system) to support real-time energy flexibility apps, rapidly and scalable. diff --git a/documentation/dev/docker-compose.rst b/documentation/dev/docker-compose.rst index 1bbc8862a..884e882f1 100644 --- a/documentation/dev/docker-compose.rst +++ b/documentation/dev/docker-compose.rst @@ -3,7 +3,7 @@ Running a complete stack with docker-compose ============================================= -Installing FlexMeasures and dependencies is some work, and can have unexpected hurdles, e.g. depending on the operating system. So, running the whole stack via `Docker compose `_ can be a very useful option. +To install FlexMeasures, plus the libraries and databases it depends on, on your computer is some work, and can have unexpected hurdles, e.g. depending on the operating system. A nice alternative is to let that happen within Docker. The whole stack can be run via `Docker compose `_, saving the developer much time. For this, we assume you are in the directory housing ``docker-compose.yml``. diff --git a/documentation/host/docker.rst b/documentation/host/docker.rst index 7af737aba..730499558 100644 --- a/documentation/host/docker.rst +++ b/documentation/host/docker.rst @@ -3,7 +3,7 @@ Running via Docker ====================== -FlexMeasures can be run via `docker `_. +FlexMeasures can be run via `docker `_. `Docker `_ is great to save developers from installation trouble, but also for running FlexMeasures inside modern cloud environments in a scalable manner. For now, the use case is local development. Using in production is a goal for later. @@ -48,7 +48,7 @@ Running the image (as a container) might work like this (remember to get the ima .. note:: Don't know what your image is called (its "tag")? We used ``lfenergy/flexmeasures`` here, as that should be the name when pulling it from Docker Hub. You can run ``docker images`` to see which images you have. -The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``Flask_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). If you see too much output, you can also set ``LOGGING_LEVEL=INFO``. +The two minimal environment variables to run the container successfully are the database URI and the secret key, see :ref:`configuration`. ``FLASK_ENV=development`` is needed if you do not have an SSL certificate set up (the default mode is ``production``, and in that mode FlexMeasures requires https for security reasons). If you see too much output, you can also set ``LOGGING_LEVEL=INFO``. In this example, we connect to a postgres database running on our local computer, so we use the host network. In the docker-compose section below, we use a Docker container for the database, as well. diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 5c7ddd34b..7220cd62f 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -46,8 +46,8 @@ Install Flexmeasures and the database $ docker pull lfenergy/flexmeasures:latest $ docker pull postgres - $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest - $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:docker@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host lfenergy/flexmeasures + $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=fm-db-passwd -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest + $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:fm-db-passwd@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host lfenergy/flexmeasures $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade" Now the rest of this tutorial will happen inside the FlexMeasures container. This is how you hop inside the container and run a terminal there: @@ -75,8 +75,8 @@ Install Flexmeasures and the database .. code-block:: console sudo -i -u postgres - createdb -U postgres flexmeasures - createuser --pwprompt -U postgres flexmeasures # enter your password, we'll use "flexmeasures" + createdb -U postgres flexmeasures-db + createuser --pwprompt -U postgres flexmeasures-user # enter your password, we'll use "fm-db-passwd" exit Then, we can install FlexMeasures itself, set some variables and tell FlexMeasures to create all tables: @@ -84,7 +84,7 @@ Install Flexmeasures and the database .. code-block:: console $ pip install flexmeasures - $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures:flexmeasures@127.0.0.1:5432/flexmeasures" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0 + $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures-user:fm-db-passwd@localhost:5432/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0 $ flexmeasures db upgrade .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. MacOs, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which uses a prepared Linux image, so it'll definitely work. diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index dfaeab002..0eeaa7868 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -943,7 +943,7 @@ def add_toy_account(kind: str, name: str): email = "toy-user@flexmeasures.io" user = User.query.filter_by(email=email).one_or_none() if user is not None: - print( + click.echo( f"User with email {email} already exists in account {user.account.name}." ) else: From 48e7c070c1eb559e2774512d9b7ef24743ee1b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 23 Apr 2022 10:13:03 +0200 Subject: [PATCH 27/30] demanding the lowest docker compose version we know we need (we use start_period for healthchecks) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 272d06295..5c4b420d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.8" +version: "3.4" # ----------------------------------------------- # TODO: From 555dfd5456f71a91f9c3cde737358da8a1e79ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 23 Apr 2022 12:50:48 +0200 Subject: [PATCH 28/30] more attention-grabbing note about the tutorial happening inside the docker container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/tut/toy-example-from-scratch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 7220cd62f..031df4e97 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -50,7 +50,7 @@ Install Flexmeasures and the database $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:fm-db-passwd@localhost:5433/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --net=host lfenergy/flexmeasures $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade" - Now the rest of this tutorial will happen inside the FlexMeasures container. This is how you hop inside the container and run a terminal there: + Now - what's *very important* to remember is this: The rest of this tutorial will happen *inside* the ``flexmeasures-tutorial-fm`` container! This is how you hop inside the container and run a terminal there: .. code-block:: console From 45b88cffc82535b462322b7a8f4604ac80136c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 25 Apr 2022 13:22:05 +0200 Subject: [PATCH 29/30] not only the API runs in the container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 63b1d518b..c7f8814c1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -16,7 +16,7 @@ Bugfixes Infrastructure / Support ---------------------- -* Dockerfile to run FlexMeasures API in container; also docker-compose file [see `PR #416 `_] +* Dockerfile to run FlexMeasures in container; also docker-compose file [see `PR #416 `_] * Unit conversion prefers shorter units in general [see `PR #415 `_] * Shorter CI builds in Github Actions by caching Python environment [see `PR #361 `_] @@ -38,7 +38,6 @@ Bugfixes * Fix filter for selecting one deterministic belief per event, which was duplicating index levels [see `PR #414 `_] - v0.9.1 | March 31, 2022 =========================== From ad15a9d9bedd34ee241f08cf375c39cdeba8dcd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 25 Apr 2022 13:26:29 +0200 Subject: [PATCH 30/30] better name for the Docker container in the compose tutorial & smaller doc fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docker-compose.yml | 2 +- documentation/dev/docker-compose.rst | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5c4b420d9..5c6862e07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: POSTGRES_PASSWORD: fm-dev-db-pass volumes: - ./ci/load-psql-extensions.sql:/docker-entrypoint-initdb.d/load-psql-extensions.sql - flexmeasures: + server: build: context: . dockerfile: Dockerfile diff --git a/documentation/dev/docker-compose.rst b/documentation/dev/docker-compose.rst index 884e882f1..6055df03e 100644 --- a/documentation/dev/docker-compose.rst +++ b/documentation/dev/docker-compose.rst @@ -8,6 +8,8 @@ To install FlexMeasures, plus the libraries and databases it depends on, on your For this, we assume you are in the directory housing ``docker-compose.yml``. +.. note:: The minimum Docker version is 17.09 and for docker-compose we tested successfully at version 1.25. You can check your versions with ``docker[-compose] --version``. + Build the compose stack ------------------------ @@ -21,7 +23,7 @@ This pulls the images you need, and re-builds the FlexMeasures one from code. If This compose script can also serve as an inspiration for using FlexMeasures in modern cloud environments (like Kubernetes). For instance, you might want to not build the FlexMeasures image from code, but simply pull the image from DockerHub. -.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418`_). +.. todo:: This stack runs FlexMeasures, but misses the background worker aspect. For this, we'll add a redis node and one additional FlexMeasures node, which runs a worker as entry point instead (see `issue 418 `_). Run the compose stack @@ -41,9 +43,10 @@ Check ``docker ps`` or ``docker-compose ps`` to see if your containers are runni .. code-block:: console ± docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 6105f6d1c91f flexmeasures_flexmeasures "bash -c 'flexmeasur…" 45 seconds ago Up 44 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_flexmeasures_1 - b48e4b9b113b postgres "docker-entrypoint.s…" 44 hours ago Up 45 seconds 5432/tcp flexmeasures_dev-db_1 + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + dda1a8606926 flexmeasures_server "bash -c 'flexmeasur…" 43 seconds ago Up 41 seconds (healthy) 0.0.0.0:5000->5000/tcp flexmeasures_server_1 + 27ed9eef1b04 postgres "docker-entrypoint.s…" 2 days ago Up 42 seconds 5432/tcp flexmeasures_dev-db_1 + 90df2065e08d postgres "docker-entrypoint.s…" 2 days ago Up 42 seconds 5432/tcp flexmeasures_test-db_1 The FlexMeasures container has a health check implemented, which is reflected in this output and you can see which ports are available on your machine to interact. @@ -63,18 +66,18 @@ Data ------- The postgres database is a test database with toy data filled in when the flexmeasures container starts. -You could also connect it to some other database, by setting a different `SQLALCHEMY_DATABASE_URI` in the config. +You could also connect it to some other database, by setting a different ``SQLALCHEMY_DATABASE_URI`` in the config. Running tests --------------- -You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file. Per default, we are using the ``dev-db`` database service. +You can run tests in the flexmeasures docker container, using the database service ``test-db`` in the compose file (per default, we are using the ``dev-db`` database service). After you've started the compose stack with ``docker-compose up``, run: .. code-block:: console - docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" pytest + docker exec -it -e SQLALCHEMY_TEST_DATABASE_URI="postgresql://fm-test-db-user:fm-test-db-pass@test-db:5432/fm-test-db" flexmeasures_server_1 pytest This rounds up the dev experience offered by running FlexMeasures in Docker. Now you can develop FlexMeasures and also run your tests. If you develop plugins, you could extend the command being used, e.g. ``bash -c "cd /path/to/my/plugin && pytest"``. \ No newline at end of file