From 3370e622af87a3b8500ab5c216a04e62996d029d Mon Sep 17 00:00:00 2001 From: nhoening Date: Fri, 7 May 2021 20:26:06 +0000 Subject: [PATCH 01/11] Create draft PR for #50 From 5beb332403d6f9e421578048644e5fceeb3078cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 8 May 2021 00:29:49 +0200 Subject: [PATCH 02/11] add empty access to some 2_0 endpoints - this was breaking the getServices endpoint --- flexmeasures/api/v2_0/routes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/routes.py b/flexmeasures/api/v2_0/routes.py index 60cc27f07..a89270880 100644 --- a/flexmeasures/api/v2_0/routes.py +++ b/flexmeasures/api/v2_0/routes.py @@ -41,6 +41,7 @@ v2_0_service_listing["services"].append( { "name": "GET /asset/", + "access": [], "description": "Get an asset.", }, ) @@ -69,18 +70,21 @@ v2_0_service_listing["services"].append( { "name": "GET /user/", + "access": [], "description": "Get a user.", }, ) v2_0_service_listing["services"].append( { "name": "PATCH /user/", + "access": [], "description": "Edit a user.", }, ) v2_0_service_listing["services"].append( { "name": "PATCH /user//password-reset", + "access": [], "description": "Reset a user's password.", }, ) @@ -477,7 +481,7 @@ def reset_user_password(id: int): .. :quickref: User; Password reset Reset the user's password, and send them instructions on how to reset the password. - This endoint is useful from a security standpoint, in case of worries the password might be compromised. + This endpoint is useful from a security standpoint, in case of worries the password might be compromised. It sets the current password to something random, invalidates cookies and auth tokens, and also sends an email for resetting the password to the user. From 7cc6361d61cf62e377ce55c5a4bf6981dbb54341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 8 May 2021 00:37:19 +0200 Subject: [PATCH 03/11] improve getService and getAuthToken part of API introduction --- documentation/api/introduction.rst | 69 ++++- documentation/api/simulation.rst | 452 ----------------------------- 2 files changed, 57 insertions(+), 464 deletions(-) delete mode 100644 documentation/api/simulation.rst diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 6c58cfc0a..bf484ee49 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -7,26 +7,60 @@ This document details the Application Programming Interface (API) of the FlexMea We assume in this document that the FlexMeasures instance you want to connect to is hosted at https://company.flexmeasures.io. -New versions of the API are released on: +.. _api_versions: + +Main endpoint and API versions +------------------------------ + +All versions of the API are released on: + +.. code-block:: html + + https:///api + +So if you are running FlexMeasures on your computer, it would be: + +.. code-block:: html + + https://localhost:5000/api + +At Seita, we run servers for our clients at: .. code-block:: html https://company.flexmeasures.io/api -A list of services offered by (a version of) the FlexMeasures web service can be obtained by sending a *getService* request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. +where `company` is a hosting customer of ours. All their accounts' data lives on that server. + +Let's see what it says: + +.. code-block:: python + + >>> import requests + >>> res = requests.get("https://seita.flexmeasures.io/api") + >>> res.json() + {'flexmeasures_version': '0.4.0', 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: /api/requestAuthToken', 'status': 200, 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0']} + +So this tells us which API versions exist. For instance, we know that the latest API version is available at + +.. code-block:: html + + https://seita.flexmeasures.io/api/v2_0 + + +Also , we see that a list of services offered by (a version of) the FlexMeasures web service can be obtained by sending a *getService* request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. **Example request** -.. code-block:: json +Let's ask which endpoints are available for meter data companies (MDC): - { - "type": "GetServiceRequest", - "version": "1.0" - } +.. code-block:: html -**Example response** + https://seita.flexmeasures.io/api/v2_0/getService?access=MDC +**Example response** + .. code-block:: json { @@ -46,6 +80,8 @@ A list of services offered by (a version of) the FlexMeasures web service can be ] } +.. _api_auth: + Authentication -------------- @@ -78,6 +114,15 @@ using the following JSON message for the POST request data: "password": "" } +which gives a response like this if the credentials are correct: + +.. code-block:: json + + { + "auth_token": "", + "user_id": "" + } + .. note:: Each access token has a limited lifetime, see :ref:`auth`. @@ -339,8 +384,8 @@ These fields denote that the data should have been recorded at least 6 hours bef .. _prognoses: -Prognoses -^^^^^^^^^ +Knowledge time of Prognoses +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some POST endpoints have two optional fields to allow setting the time at which beliefs are recorded explicitly. This is useful to keep an accurate history of what was known at what time, especially for prognoses. @@ -425,8 +470,8 @@ Valid units for timeseries data in version 1 of the API are "MW" only. .. _signs: -Signs -^^^^^ +Signs of values +^^^^^^^^^^^^^^^ USEF recommends to use positive power values to indicate consumption and negative values to indicate production, i.e. to take the perspective of the Prosumer. diff --git a/documentation/api/simulation.rst b/documentation/api/simulation.rst deleted file mode 100644 index a06fbf8a8..000000000 --- a/documentation/api/simulation.rst +++ /dev/null @@ -1,452 +0,0 @@ -.. _simulation: - -Simulation -========== - -This document details examples for using a FlexMeasures server for simulation. -The API on a server that is set up for simulation is extended with several features that make it possible to run simulations of energy flows and control actions. -Please read the :ref:`api_introduction` for explanations of the message fields, specifically regarding: - -- The sign of values (:ref:`signs`) -- Valid durations (:ref:`resolutions`) -- Valid horizons (:ref:`prognoses`) -- Valid units (:ref:`simulation`) - -.. contents:: Table of contents - :local: - :depth: 1 - -Setting up ----------- - -Researchers require an admin account to set up a new simulation with a number of assets. - -Creating assets and owners -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -New assets can be created through the UI on: - -.. code-block:: html - - https://company.flexmeasures.io/assets/new - - -We recommend that researchers choose their own admin account as the asset's owner. -This way, the simulation will only require refreshing of the access token for the admin account. -Alternatively, researchers can set up unique accounts for each agent in a multi-agent simulation by creating new owners. -In this case, access tokens need to be refreshed by each agent separately. - -Authentication -^^^^^^^^^^^^^^ - -Service usage is only possible with a user authentication token specified in the request header, for example: - -.. code-block:: json - - { - "Authorization": "" - } - -The "" can be obtained on your profile after logging in: - -.. code-block:: html - - https://company.flexmeasures.io/account - -For security reasons, tokens expire after a certain amount of time (see :ref:`_auth`). -To automate token renewal, use the following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api/requestAuthToken - -Providing applicable user credentials: - -.. code-block:: json - - { - "email": "", - "password": "" - } - -Posting weather data --------------------- - -Weather data (both observations and forecasts) can be posted to the following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//postWeatherData - -Weather data can be posted for the following three types of weather sensors: - -- "radiation" (with kW/m² as unit) -- "temperature" (with °C as unit) -- "wind_speed" (with m/s as unit) - -The sensor type is part of the unique entity address for each sensor, together with the sensor's latitude and longitude. - -This "PostWeatherDataRequest" message posts temperature forecasts for 15-minute intervals between 3.00pm and 4.30pm for a weather sensor located at latitude 33.4843866 and longitude 126.477859. -The forecasts were made at noon. - -.. code-block:: json - - { - "type": "PostWeatherDataRequest", - "sensor": "ea1.2018-06.io.flexmeasures.company:temperature:33.4843866:126.477859", - "values": [ - 20.04, - 20.23, - 20.41, - 20.51, - 20.55, - 20.57 - ], - "start": "2015-01-01T15:00:00+09:00", - "duration": "PT1H30M", - "prior": "2015-01-01T12:00:00+09:00", - "unit": "°C" - } - -Observations vs forecasts -^^^^^^^^^^^^^^^^^^^^^^^^^ - -To post an observation rather than a forecast, simply set the prior to the moment at which the observations were made, e.g. at "2015-01-01T16:30:00+09:00". -This denotes that the observation was made exactly after realisation of this list of temperature readings, i.e. at 4.30pm. - -Alternatively, to indicate that each individual observation was made directly after the end of its 15-minute interval (i.e. at 3.15pm, 3.30pm and so on), set a horizon to "PT0H" instead of a prior. - -Finally, delays in reading out sensor data can be simulated by setting the horizon field to a negative value. -For example, a horizon of "-PT1H" would denote that each temperature reading was observed one hour after the fact (i.e. at 4.15pm, 4.30 pm and so on). - -See :ref:`prognoses` for more information regarding the prior and horizon fields. - - -Posting price data ------------------- - -Price data (both observations and forecasts) can be posted to the following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//postPriceData - -This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day -for the Korean Power Exchange (KPX) day-ahead auction. -The horizon indicates that the prices were published at 3pm on December 31st 2014 -(i.e. 33 hours ahead of midnight the next day). - -.. code-block:: json - - { - "type": "PostPriceDataRequest", - "market": "ea1.2018-06.io.flexmeasures.company:kpx_da", - "values": [ - 52.37, - 51.14, - 49.09, - 48.35, - 48.47, - 49.98, - 58.7, - 67.76, - 69.21, - 70.26, - 70.46, - 70, - 70.7, - 70.41, - 70, - 64.53, - 65.92, - 69.72, - 70.51, - 75.49, - 70.35, - 70.01, - 66.98, - 58.61 - ], - "start": "2015-01-01T15:00:00+09:00", - "duration": "PT24H", - "horizon": "PT33H", - "unit": "KRW/kWh" - } - -Observations vs forecasts -^^^^^^^^^^^^^^^^^^^^^^^^^ - -For markets, the time at which the market is cleared (i.e. when contracts are signed) determines the difference between an ex-post observation and an ex-ante forecast. -For example, at the KPX day-ahead auction this is every day at 3pm. -To post a forecast rather than an observation, simply increase the horizon. -For example, a horizon of "PT57H" would denote a forecast of 24 hours ahead of clearing. - - -Posting power data ------------------- - -For power data, USEF specifies separate message types for observations and forecasts. -Correspondingly, FlexMeasures uses separate endpoints to communicate these messages. -Observations of power data can be posted to the following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//postMeterData - -while forecasts of power data can be posted to the following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//postPrognosis - -For both endpoints, power data can be posted in various ways. -The following examples assume that the endpoint for power data observations (i.e. meter data) is used. - - -Single value, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A single average power value for a 15-minute time interval for a single connection, posted 5 minutes after realisation. - -.. code-block:: json - - { - "type": "PostMeterDataRequest", - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "value": 220, - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } - -Multiple values, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after realisation. - -.. code-block:: json - - { - "type": "PostMeterDataRequest", - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "values": [ - 220, - 210, - 200 - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H45M", - "horizon": "-PT5M", - "unit": "MW" - } - -Single identical value, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Single identical value for a 15-minute time interval for two connections, posted 5 minutes after realisation. -Please note that both connections consumed at 10 MW, i.e. the value does not represent the total of the two connections. -We recommend to use this notation for zero values only. - -.. code-block:: json - - { - "type": "PostMeterDataRequest", - "connections": [ - "ea1.2018-06.io.flexmeasures.company:1:1", - "ea1.2018-06.io.flexmeasures.company:1:2" - ], - "value": 10, - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } - -Single different values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Single different values for a 15-minute time interval for two connections, posted 5 minutes after realisation. - -.. code-block:: json - - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "value": 220 - }, - { - "connection": "ea1.2018-06.io.flexmeasures.company:1:2", - "value": 300 - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } - -Multiple values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after realisation. - -.. code-block:: json - - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "values": [ - 220, - 210, - 200 - ] - }, - { - "connection": "ea1.2018-06.io.flexmeasures.company:1:2", - "values": [ - 300, - 303, - 306 - ] - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H45M", - "horizon": "-PT5M", - "unit": "MW" - } - -Getting prognoses ------------------ - -Prognoses are power forecasts that are used by FlexMeasures to determine the best control signals to valorise on -balancing opportunities. Researchers can check the accuracy of these forecasts by downloading the prognoses and -comparing them against the meter data, i.e. the realised power measurements. -A prognosis can be requested for a single asset at the following GET endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//getPrognosis - -This example requests a prognosis with a rolling horizon of 6 hours before realisation. - -.. code-block:: json - - { - "type": "GetPrognosisRequest", - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT24H", - "horizon": "PT6H", - "resolution": "PT15M", - "unit": "MW" - } - -Posting flexibility constraints -------------------------------- - -Prosumers that have Active Demand & Supply can post the constraints of their flexible devices to FlexMeasures at the -following POST endpoint: - -.. code-block:: html - - https://company.flexmeasures.io/api//postUdiEvent - -This example posts a state of charge value for a battery device (asset 10 of owner 7) as UDI event 203. - -.. code-block:: json - - { - "type": "PostUdiEventRequest", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc", - "value": 12.1, - "datetime": "2015-06-02T10:00:00+00:00", - "unit": "kWh" - } - -Some devices also accept target values for their state of charge. -As an example, consider the same UDI event as above with an additional target value. - -.. code-block:: json - - { - "type": "PostUdiEventRequest", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:204:soc-with-targets", - "value": 12.1, - "datetime": "2015-06-02T10:00:00+00:00", - "unit": "kWh", - "targets": [ - { - "value": 25, - "datetime": "2015-06-02T16:00:00+00:00" - } - ] - } - -Getting control signals ------------------------ - -A Prosumer can query FlexMeasures for control signals for its flexible devices using the following GET endpoint: - - -.. code-block:: html - - https://company.flexmeasures.io/api//getDeviceMessage - -Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. -This example requests a control signal for UDI event 203 posted previously. - -.. code-block:: json - - { - "type": "GetDeviceMessageRequest", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc" - } - -The following example response indicates that FlexMeasures planned ahead 45 minutes. -The list of consecutive power values represents the target consumption of the battery (negative values for production). -Each value represents the average power over a 15 minute time interval. - -.. sourcecode:: json - - { - "type": "GetDeviceMessageResponse", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:203", - "values": [ - 2.15, - 3, - 2 - ], - "start": "2015-06-02T10:00:00+00:00", - "duration": "PT45M", - "unit": "MW" - } - -One way of reaching the target consumption in this example is to let the battery start to consume with 2.15 MW at 10am, -increase its consumption to 3 MW at 10.15am and decrease its consumption to 2 MW at 10.30am. -However, because the targets values represent averages over 15-minute time intervals, the battery still has some degrees of freedom. -For example, the battery might start to consume with 2.1 MW at 10.00am and increase its consumption to 2.25 at 10.10am, -increase its consumption to 5 MW at 10.15am and decrease its consumption to 2 MW at 10.20am. -That should result in the same average values for each quarter-hour. - -Resetting the server --------------------- - -All power, price and weather data on the simulation server can be cleared using the following PUT endpoint (admin rights are required): - -.. code-block:: html - - https://company.flexmeasures.io/api//restoreData - -This example restores the database to a backup named demo_v0, which contains no timeseries data. - -.. code-block:: json - - { - "backup": "demo_v0" - } From ddad8240b10258c77be0fb26df454cdc6f39ee0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 8 May 2021 00:38:00 +0200 Subject: [PATCH 04/11] move simulation part to new tutorial folder and rewrite the beginning --- documentation/changelog.rst | 1 + documentation/index.rst | 7 +- documentation/int/introduction.rst | 59 --- documentation/scripts/init_docs.py | 91 ---- .../scripts/init_docs_with_selenium.py | 71 ---- documentation/tut/posting_data.rst | 400 ++++++++++++++++++ 6 files changed, 407 insertions(+), 222 deletions(-) delete mode 100644 documentation/int/introduction.rst delete mode 100755 documentation/scripts/init_docs.py delete mode 100644 documentation/scripts/init_docs_with_selenium.py create mode 100644 documentation/tut/posting_data.rst diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b01fc0502..746b7455f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,6 +18,7 @@ Bugfixes Infrastructure / Support ---------------------- * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] +* Add tutorials on how to add and read data from FlexMeasures via its API [see `PR #130 `_] v0.4.1 | May 7, 2021 diff --git a/documentation/index.rst b/documentation/index.rst index b05c75b0c..0f95993a9 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -45,6 +45,12 @@ The platform operator of FlexMeasures can be an Aggregator. concepts/security_auth +.. toctree:: + :caption: Tutorials + :maxdepth: 1 + + tut/posting_data + .. toctree:: :caption: The in-built UI :maxdepth: 1 @@ -60,7 +66,6 @@ The platform operator of FlexMeasures can be an Aggregator. :maxdepth: 1 api/introduction - api/simulation api/v2_0 api/v1_3 api/v1_2 diff --git a/documentation/int/introduction.rst b/documentation/int/introduction.rst deleted file mode 100644 index 9bd70cee4..000000000 --- a/documentation/int/introduction.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _integrations_introduction: - -Introduction -============ - -The platform's functionality can be integrated into any third-party workflow by using the :ref:`API `. -In this section we discuss several examples of third-party IoT software for integrating with hardware devices. -These systems may be considered especially useful for Aggregators, Energy Service Companies and Prosumers. -Third-party software for other stakeholders (such as Suppliers, DSOs and TSOs) are proprietary systems for which open documentation is not adequately available. - -.. _home_assistant: - -Home Assistant -============== - -Home Assistant is an open-source system for hardware-software integration, supporting >1200 hardware components and software services. -The system is written in Python, designed to run on a lightweight local server (such as a Raspberry Pi) and geared towards the residential sector. - -.. code-block:: html - - https://home-assistant.io - -One of the supported integrations is with Z-Wave components such as the Z-Wave Aeotec clamp power meter. - -.. code-block:: html - - https://aeotec.com/z-wave-home-energy-measure - -To collect meter data from individual devices, the local server should be fitted with a Z-stick. -An advantage of Z-Wave wireless communication technology is that it operates as a mesh network. -This means that adding a Z-Wave component to the local system extends the system's communication range. -The maximum range of individual Z-Wave components is approximately 30 meters. - -.. _smappee: - -Smappee -======= - -Smappee is a proprietary system providing a suite of services supporting full hardware-software integration. -The system is geared towards the residential sector. - -.. code-block:: html - - https://smappee.com - -Products include sub-metering using power clamps, plug-and-play power socket actuators and analysis dashboards. -In addition, it offers software integrations with several other smart home products and services, -such as a number of brands of thermostats and air conditioning. - -Smappee may be integrated with the BPV platform using Smappy, -a Python wrapper for the Smappee API, which can retrieve sensor data and control actuators. - -.. code-block:: html - - https://github.com/EnergieID/smappy/wiki - -The API can be used to retrieve meter data (with a specific resolution) of individual devices using the `get_sensor_consumption` function. -This data can then be sent to FlexMeasures using the `post_meter_data` endpoint. -In addition, device messages from FlexMeasures intended to curtail consumption (for a specific duration) can be sent to individual devices using the `actuator_off` function. diff --git a/documentation/scripts/init_docs.py b/documentation/scripts/init_docs.py deleted file mode 100755 index 868bba4ce..000000000 --- a/documentation/scripts/init_docs.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -""" -Script for initialising the documentation -""" -import os -from subprocess import call - -import sys -import time -from PyQt5.QtCore import QSize, QUrl -from PyQt5.QtGui import QImage, QPainter -from PyQt5.QtWebKitWidgets import QWebView -from PyQt5.QtWidgets import QApplication - - -path_to_doc = "documentation" -if os.getcwd().endswith("scripts"): - path_to_doc = ".." - -make_docs_cmd = "make html" -if os.name != "posix": - # here we re-activate the virtual environment first - make_docs_cmd = "activate a1-venv & " + make_docs_cmd - - -class ScreenShot(QWebView): - def __init__(self): - self.app = QApplication(sys.argv) - QWebView.__init__(self) - self._loaded = False - self.loadFinished.connect(self.load_finished) - - def capture(self, url, output_file, width=1500, height=1000): - self.load(QUrl(url)) - self.wait_load() - # set to webpage size - frame = self.page().mainFrame() - self.page().setViewportSize(QSize(width, height)) - # render image - time.sleep(5) - image = QImage(self.page().viewportSize(), QImage.Format_ARGB32) - painter = QPainter(image) - frame.render(painter) - painter.end() - print("saving to ", output_file) - image.save(output_file) - - def wait_load(self, delay=0): - # process app events until page loaded - while not self._loaded: - self.app.processEvents() - time.sleep(delay) - self._loaded = False - - def load_finished(self, result): - self._loaded = True - - -def make_screen_shots(views): - - print("Capturing screen shots ...") - - width = 1500 - height = 1500 - s = ScreenShot() - for view in views: - url = "http://127.0.0.1:5000/%s" % view - print("Loading", url) - s.capture( - url, - path_to_doc + "/img/screenshot_%s.png" % view, - width=width, - height=height, - ) - return - - -def initialise_docs(): - """Initialise doc files""" - - print("Processing documentation files ...") - call(make_docs_cmd, shell=True, cwd=path_to_doc) - - -if __name__ == "__main__": - """Initialise screen shots and documentation""" - # Todo: log in before taking screenshots - - # make_screen_shots(["dashboard", "portfolio", "control", "analytics"]) - initialise_docs() diff --git a/documentation/scripts/init_docs_with_selenium.py b/documentation/scripts/init_docs_with_selenium.py deleted file mode 100644 index c51f44bbf..000000000 --- a/documentation/scripts/init_docs_with_selenium.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -""" -Script for initialising the documentation -""" -import os -from slugify import slugify -from subprocess import call -from selenium import webdriver -from PIL import Image, ImageOps - -path_to_doc = "documentation" -path_return = ".." -if os.getcwd().endswith("scripts"): - path_to_doc = "../documentation" - path_return = "../scripts" - -make_docs_cmd = "cd %s; make html; cd %s" % (path_to_doc, path_return) -if os.name != "posix": - make_docs_cmd = make_docs_cmd.replace(";", " &") - - # here we re-activate the virtual environment first - make_docs_cmd = "activate a1-venv & " + make_docs_cmd - - -def initialise_screen_shots(views): - width = 1500 - max_height = 3000 - browser = webdriver.Firefox() - browser.set_window_position(0, 0) - browser.set_window_size(width, max_height) - for view in views: - print("Saving " + view) - url = "http://127.0.0.1:5000/" + view - browser.get(url) - output_file = "../documentation/img/screenshot_" + slugify(view) + ".png" - browser.save_screenshot(output_file) - browser.close() - - # remove trailing white rows from the image - for view in views: - output_file = "../documentation/img/screenshot_" + slugify(view) + ".png" - im = Image.open(output_file) - im.crop(ImageOps.invert(im.convert("RGB")).getbbox()).save(output_file) - - return - - -def initialise_docs(): - """Initialise doc files""" - - print("Processing documentation files ...") - - call(make_docs_cmd, shell=True) - - -if __name__ == "__main__": - """Initialise screen shots and documentation""" - - initialise_screen_shots( - [ - "dashboard", - "portfolio", - "control", - "analytics", - "dashboard?prosumer_mock=vehicles", - "portfolio?prosumer_mock=vehicles", - "control?prosumer_mock=vehicles", - ] - ) - initialise_docs() diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst new file mode 100644 index 000000000..ce2973b16 --- /dev/null +++ b/documentation/tut/posting_data.rst @@ -0,0 +1,400 @@ +.. _simulation: + +Posting data +============ + +Let's demonstrate how you can get data into FlexMeasures using the API. This is where FlexMeasures gets connected to your system as a smart backend and helps you build smart energy services. + +.. contents:: Table of contents + :local: + :depth: 1 + +You should be familiar with where to find your API and how to authenticate against it (see :ref:`api_auth`). + +.. note:: You can always read the :ref:`api_introduction` for deeper explanations of content, e.g. :ref:`signs`, :ref:`resolutions`, :ref:`prognoses` and :ref:`units`. + + + +Posting weather data +-------------------- + +Weather data (both observations and forecasts) can be posted to the following POST endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//postWeatherData + +.. note:: TODO Prerequisites: ``flexmeasures add structure``, ``flexmeasures add weather-sensor`` + +Weather data can be posted for the following three types of weather sensors: + +- "radiation" (with kW/m² as unit) +- "temperature" (with °C as unit) +- "wind_speed" (with m/s as unit) + +The sensor type is part of the unique entity address for each sensor, together with the sensor's latitude and longitude. + +This "PostWeatherDataRequest" message posts temperature forecasts for 15-minute intervals between 3.00pm and 4.30pm for a weather sensor located at latitude 33.4843866 and longitude 126.477859. TODO: which is in Korea's timezone, see see iso datetimes. +The forecasts were made at noon. + +.. code-block:: json + + { + "type": "PostWeatherDataRequest", + "sensor": "ea1.2018-06.io.flexmeasures.company:temperature:33.4843866:126.477859", + "values": [ + 20.04, + 20.23, + 20.41, + 20.51, + 20.55, + 20.57 + ], + "start": "2015-01-01T15:00:00+09:00", + "duration": "PT1H30M", + "prior": "2015-01-01T12:00:00+09:00", + "unit": "°C" + } + +Observations vs forecasts +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To post an observation rather than a forecast, simply set the prior to the moment at which the observations were made, e.g. at "2015-01-01T16:30:00+09:00". +This denotes that the observation was made exactly after realisation of this list of temperature readings, i.e. at 4.30pm. + +Alternatively, to indicate that each individual observation was made directly after the end of its 15-minute interval (i.e. at 3.15pm, 3.30pm and so on), set a horizon to "PT0H" instead of a prior. + +Finally, delays in reading out sensor data can be simulated by setting the horizon field to a negative value. +For example, a horizon of "-PT1H" would denote that each temperature reading was observed one hour after the fact (i.e. at 4.15pm, 4.30 pm and so on). + +See :ref:`prognoses` for more information regarding the prior and horizon fields. + + +Posting price data +------------------ + +Price data (both observations and forecasts) can be posted to the following POST endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//postPriceData + +This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day +for the Korean Power Exchange (KPX) day-ahead auction. +The horizon indicates that the prices were published at 3pm on December 31st 2014 +(i.e. 33 hours ahead of midnight the next day). + +.. code-block:: json + + { + "type": "PostPriceDataRequest", + "market": "ea1.2018-06.io.flexmeasures.company:kpx_da", + "values": [ + 52.37, + 51.14, + 49.09, + 48.35, + 48.47, + 49.98, + 58.7, + 67.76, + 69.21, + 70.26, + 70.46, + 70, + 70.7, + 70.41, + 70, + 64.53, + 65.92, + 69.72, + 70.51, + 75.49, + 70.35, + 70.01, + 66.98, + 58.61 + ], + "start": "2015-01-01T15:00:00+09:00", + "duration": "PT24H", + "horizon": "PT33H", + "unit": "KRW/kWh" + } + +Observations vs forecasts +^^^^^^^^^^^^^^^^^^^^^^^^^ + +For markets, the time at which the market is cleared (i.e. when contracts are signed) determines the difference between an ex-post observation and an ex-ante forecast. +For example, at the KPX day-ahead auction this is every day at 3pm. +To post a forecast rather than an observation, simply increase the horizon. +For example, a horizon of "PT57H" would denote a forecast of 24 hours ahead of clearing. + + +Posting power data +------------------ + +For power data, USEF specifies separate message types for observations and forecasts. +Correspondingly, FlexMeasures uses separate endpoints to communicate these messages. +Observations of power data can be posted to the following POST endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//postMeterData + +while forecasts of power data can be posted to the following POST endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//postPrognosis + +For both endpoints, power data can be posted in various ways. +The following examples assume that the endpoint for power data observations (i.e. meter data) is used. + + +Single value, single connection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A single average power value for a 15-minute time interval for a single connection, posted 5 minutes after realisation. + +.. code-block:: json + + { + "type": "PostMeterDataRequest", + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "value": 220, + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } + +Multiple values, single connection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after realisation. + +.. code-block:: json + + { + "type": "PostMeterDataRequest", + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "values": [ + 220, + 210, + 200 + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H45M", + "horizon": "-PT5M", + "unit": "MW" + } + +Single identical value, multiple connections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Single identical value for a 15-minute time interval for two connections, posted 5 minutes after realisation. +Please note that both connections consumed at 10 MW, i.e. the value does not represent the total of the two connections. +We recommend to use this notation for zero values only. + +.. code-block:: json + + { + "type": "PostMeterDataRequest", + "connections": [ + "ea1.2018-06.io.flexmeasures.company:1:1", + "ea1.2018-06.io.flexmeasures.company:1:2" + ], + "value": 10, + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } + +Single different values, multiple connections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Single different values for a 15-minute time interval for two connections, posted 5 minutes after realisation. + +.. code-block:: json + + { + "type": "PostMeterDataRequest", + "groups": [ + { + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "value": 220 + }, + { + "connection": "ea1.2018-06.io.flexmeasures.company:1:2", + "value": 300 + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } + +Multiple values, multiple connections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after realisation. + +.. code-block:: json + + { + "type": "PostMeterDataRequest", + "groups": [ + { + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "values": [ + 220, + 210, + 200 + ] + }, + { + "connection": "ea1.2018-06.io.flexmeasures.company:1:2", + "values": [ + 300, + 303, + 306 + ] + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H45M", + "horizon": "-PT5M", + "unit": "MW" + } + +Getting prognoses +----------------- + +Prognoses are power forecasts that are used by FlexMeasures to determine the best control signals to valorise on +balancing opportunities. Researchers can check the accuracy of these forecasts by downloading the prognoses and +comparing them against the meter data, i.e. the realised power measurements. +A prognosis can be requested for a single asset at the following GET endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//getPrognosis + +This example requests a prognosis with a rolling horizon of 6 hours before realisation. + +.. code-block:: json + + { + "type": "GetPrognosisRequest", + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT24H", + "horizon": "PT6H", + "resolution": "PT15M", + "unit": "MW" + } + +Posting flexibility constraints +------------------------------- + +Prosumers that have Active Demand & Supply can post the constraints of their flexible devices to FlexMeasures at the +following POST endpoint: + +.. code-block:: html + + https://company.flexmeasures.io/api//postUdiEvent + +This example posts a state of charge value for a battery device (asset 10 of owner 7) as UDI event 203. + +.. code-block:: json + + { + "type": "PostUdiEventRequest", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc", + "value": 12.1, + "datetime": "2015-06-02T10:00:00+00:00", + "unit": "kWh" + } + +Some devices also accept target values for their state of charge. +As an example, consider the same UDI event as above with an additional target value. + +.. code-block:: json + + { + "type": "PostUdiEventRequest", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:204:soc-with-targets", + "value": 12.1, + "datetime": "2015-06-02T10:00:00+00:00", + "unit": "kWh", + "targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + } + ] + } + +Getting control signals +----------------------- + +A Prosumer can query FlexMeasures for control signals for its flexible devices using the following GET endpoint: + + +.. code-block:: html + + https://company.flexmeasures.io/api//getDeviceMessage + +Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. +This example requests a control signal for UDI event 203 posted previously. + +.. code-block:: json + + { + "type": "GetDeviceMessageRequest", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc" + } + +The following example response indicates that FlexMeasures planned ahead 45 minutes. +The list of consecutive power values represents the target consumption of the battery (negative values for production). +Each value represents the average power over a 15 minute time interval. + +.. sourcecode:: json + + { + "type": "GetDeviceMessageResponse", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:203", + "values": [ + 2.15, + 3, + 2 + ], + "start": "2015-06-02T10:00:00+00:00", + "duration": "PT45M", + "unit": "MW" + } + +One way of reaching the target consumption in this example is to let the battery start to consume with 2.15 MW at 10am, +increase its consumption to 3 MW at 10.15am and decrease its consumption to 2 MW at 10.30am. +However, because the targets values represent averages over 15-minute time intervals, the battery still has some degrees of freedom. +For example, the battery might start to consume with 2.1 MW at 10.00am and increase its consumption to 2.25 at 10.10am, +increase its consumption to 5 MW at 10.15am and decrease its consumption to 2 MW at 10.20am. +That should result in the same average values for each quarter-hour. + +Resetting the server +-------------------- + +All power, price and weather data on the simulation server can be cleared using the following PUT endpoint (admin rights are required): + +.. code-block:: html + + https://company.flexmeasures.io/api//restoreData + +This example restores the database to a backup named demo_v0, which contains no timeseries data. + +.. code-block:: json + + { + "backup": "demo_v0" + } From 9b9cac1214f2067e986e520eba3abce556af5767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 9 May 2021 00:14:16 +0200 Subject: [PATCH 05/11] first version of posting-data tutorial, prepare two others: forecasting/scheduling and custom-UIs --- documentation/getting-started.rst | 42 +----- documentation/tut/building-uis.rst | 10 ++ documentation/tut/forecasting_scheduling.rst | 126 ++++++++++++++++++ documentation/tut/posting_data.rst | 130 +++++-------------- 4 files changed, 174 insertions(+), 134 deletions(-) create mode 100644 documentation/tut/building-uis.rst create mode 100644 documentation/tut/forecasting_scheduling.rst diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 4bbf9d6d7..8d8634d7d 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -8,6 +8,9 @@ Quickstart This section walks you through getting FlexMeasures to run with the least effort. We'll cover making a secret key, connecting a database and creating one user & one asset. +.. note: Are you not hosting FlexMeasures, but wanting to learn how to use it? head over to our tutorials, starting with :ref:`_tut_posting_data`. + + Install FlexMeasures ^^^^^^^^^^^^^^^^^^^^ @@ -134,7 +137,7 @@ Add your first asset There are three ways to add assets: -Head over to ``http://localhost:5000/assets`` and add a new asset there. +Head over to ``http://localhost:5000/assets`` (after you started FlexMeasures, see next step) and add a new asset there in a web form. Or, use the ``flexmeasures`` :ref:`cli`: @@ -208,40 +211,3 @@ Installing Cbc can be done on Unix via: We provide a script for installing from source (without requiring ``sudo`` rights) in :ref:`continuous_integration`. More information (e.g. for installing on Windows) on `the Cbc website `_. - -Start collecting weather data -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To collect weather measurements and forecasts from the DarkSky API, there is a task you could run periodically, probably once per hour. Here is an example: - -.. code-block:: - - flexmeasures add external-weather-forecasts --location 33.4366,126.5269 --store-in-db - -.. note:: DarkSky is not handing out tokens any more, as they have been bought by Apple (see `issue 3 `_). - - -Preparing the job queue database and start workers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To let FlexMeasures queue forecasting and scheduling jobs, install a `Redis `_ server and configure access to it within FlexMeasures' config file (see above). You can find the necessary settings in :ref:`redis-config`. - -Then run one worker for each kind of job (in a separate terminal): - -.. code-block:: - - flexmeasures run-worker --queue forecasting - flexmeasures run-worker --queue scheduling - - -You can also clear the job queues: - -.. code-block:: - - flexmeasures clear-queue --queue forecasting - flexmeasures clear-queue --queue scheduling - - -When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``\ , respectively (by admins). - -When forecasts and schedules have been generated, they should be visible at ``http://localhost:5000/analytics``. You can also access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis `_\ , and schedules via `GET /api/v2_0/getDeviceMessage `_. diff --git a/documentation/tut/building-uis.rst b/documentation/tut/building-uis.rst new file mode 100644 index 000000000..f0d0f0b33 --- /dev/null +++ b/documentation/tut/building-uis.rst @@ -0,0 +1,10 @@ +.. _tut_building_uis: + +Building UIs on top of the FlexMeasures API +======================== + +TODO: Explain why using FlexMeasure information in custom UIs might be beneficial. + +TODO: show three calls to our API which we didn't show yet: user info, asset info and graphs. Example with generic JavaScript how to embed in a HTML page. + + diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst new file mode 100644 index 000000000..8f80aa462 --- /dev/null +++ b/documentation/tut/forecasting_scheduling.rst @@ -0,0 +1,126 @@ +.. _tut_forecasting_scheduling: + +Forecasting & scheduling +======================== + +Once FlexMeasures has been integrated with data (see :ref:`_tut_posting_data`), you can enjoy its forecasting and scheduling services. +Let's take a look how to set this up (if you are hosting FlexMeasures yourself) and how to access this information. + +.. note: If you are not hosting FlexMeasures yourself, skip to :ref:`getting_prognoses`. + + +Maintaining the queues +------------------------------------ + +Here we assume you have access to a Redis server and configured it (see :ref:`redis-config`). + +Start to run one worker for each kind of job (in a separate terminal): + +.. code-block:: + + flexmeasures run-worker --queue forecasting + flexmeasures run-worker --queue scheduling + + +You can also clear the job queues: + +.. code-block:: + + flexmeasures clear-queue --queue forecasting + flexmeasures clear-queue --queue scheduling + + +When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the queues of forecasting and scheduling jobs can be visited at ``http://localhost:5000/tasks/forecasting`` and ``http://localhost:5000/tasks/schedules``\ , respectively (by admins). + +When forecasts and schedules have been generated, they should be visible at ``http://localhost:5000/analytics``. + + +Queueing jobs +------------------ + +TODO: Explain how forecasting jobs are made by the API when new data arrives. Scheduling jobs also, when PostUdiEvent is called. Future work: how can a user configure which assets get this treatment? + +TODO: Show that we have flexmeasures add forecasts, example: --from_date 2015-02-02 --to_date 2015-02-04 --horizon_hours 6 --asset-id 2 --as-job + Drawback: only for complete days (can we change that?) + + +.. _getting_prognoses: + +Getting forecasts (prognoses) +----------------- + +Prognoses (the USEF term used for forecasts) are used by FlexMeasures to determine the best control signals to valorise on +balancing opportunities. + +You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis `_ +Getting them might be useful if you want to use prognoses in your own system or to check the accuracy of these forecasts by downloading the prognoses and +comparing them against the meter data, i.e. the realised power measurements (though the FlexMeasures UI also visualises them next to each other). + +So a prognosis can be requested for a single asset at the ``getPrognosis`` endpoint, at an URL looking like this: + +.. code-block:: html + + https://company.flexmeasures.io/api//getPrognosis + +This example requests a prognosis for 24 hours, with a rolling horizon of 6 hours before realisation. + +.. code-block:: json + + { + "type": "GetPrognosisRequest", + "connection": "ea1.2018-06.io.flexmeasures.company:1:1", + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT24H", + "horizon": "PT6H", + "resolution": "PT15M", + "unit": "MW" + } + + +Getting schedules (control signals) +----------------------- + +FlexMeasures can create optimised schedules with control signals for flexible devices. You can access the schedules via the `GET /api/v2_0/getDeviceMessage `_ endpoint. The URL then looks like this: + +.. code-block:: html + + https://company.flexmeasures.io/api//getDeviceMessage + +Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. +This example of a request body shows that we want to look up a control signal for UDI event 203 (which was posted previously, see :ref:`posting_flex_constraints`). + +.. code-block:: json + + { + "type": "GetDeviceMessageRequest", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc" + } + +The following example response indicates that FlexMeasures planned ahead 45 minutes for this battery. +The list of consecutive power values represents the target consumption of the battery (negative values for production). +Each value represents the average power over a 15 minute time interval. + +.. sourcecode:: json + + { + "type": "GetDeviceMessageResponse", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:203", + "values": [ + 2.15, + 3, + 2 + ], + "start": "2015-06-02T10:00:00+00:00", + "duration": "PT45M", + "unit": "MW" + } + +How to interpret these control signals? + +One way of reaching the target consumption in this example is to let the battery start to consume with 2.15 MW at 10am, +increase its consumption to 3 MW at 10.15am and decrease its consumption to 2 MW at 10.30am. + +However, because the targets values represent averages over 15-minute time intervals, the battery still has some degrees of freedom. +For example, the battery might start to consume with 2.1 MW at 10.00am and increase its consumption to 2.25 at 10.10am, +increase its consumption to 5 MW at 10.15am and decrease its consumption to 2 MW at 10.20am. +That should result in the same average values for each quarter-hour. diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index ce2973b16..9924c7b69 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -1,18 +1,27 @@ -.. _simulation: +.. _tut_posting_data: Posting data ============ -Let's demonstrate how you can get data into FlexMeasures using the API. This is where FlexMeasures gets connected to your system as a smart backend and helps you build smart energy services. +The platform FlexMeasures strives on the data you feed it. Let's demonstrate how you can get data into FlexMeasures using the API. This is where FlexMeasures gets connected to your system as a smart backend and helps you build smart energy services. + +We will show how to use the API endpoints for POSTing data. You can call these in regular intervals (through scheduled scripts in your system, for example), so that FlexMeasures is always integrated with the recent data. Of course, these endpoints can also be used to load historic data into FlexMeasures, so that the forecasting models have enough data history to work with. + +.. note:: For the purposes of forecasting and scheduling, it is often advisable to use a higher resolution than most metering services keep. For example, while such services might measure every ten seconds, FlexMeasures will usually do its job no less effective if you feed it data with a resolution of five minutes. This will also make the data integration much easier. Keep also in mind that many data sources like weather forecasting or markets can have data resolutions of an hour, anyway. .. contents:: Table of contents :local: :depth: 1 -You should be familiar with where to find your API and how to authenticate against it (see :ref:`api_auth`). +Prerequisites +-------------- -.. note:: You can always read the :ref:`api_introduction` for deeper explanations of content, e.g. :ref:`signs`, :ref:`resolutions`, :ref:`prognoses` and :ref:`units`. +- FlexMeasures needs some structural meta data for data to be understood. For example, for adding weather data we need to define a weather sensor, and what kind of weather sensors there are. You also need a user account. If you host FlexMeasures yourself, you need to add this info first. Head over to :ref:`getting_started`, where these steps are covered, or study our :ref:`cli`. +- You should be familiar with where to find your API endpoints (see :ref:`api_versions`) and how to authenticate against the API (see :ref:`api_auth`). +.. note:: For deeper explanations of the data and the meta fields we'll send here, You can always read the :ref:`api_introduction` , e.g. :ref:`signs`, :ref:`resolutions`, :ref:`prognoses` and :ref:`units`. + +.. note:: To address assets and sensors, these tutorials assume entity addresses valid in the namespace ``fm0``. See :ref:`api_introduction` for more explanations. Posting weather data @@ -24,8 +33,6 @@ Weather data (both observations and forecasts) can be posted to the following PO https://company.flexmeasures.io/api//postWeatherData -.. note:: TODO Prerequisites: ``flexmeasures add structure``, ``flexmeasures add weather-sensor`` - Weather data can be posted for the following three types of weather sensors: - "radiation" (with kW/m² as unit) @@ -34,8 +41,8 @@ Weather data can be posted for the following three types of weather sensors: The sensor type is part of the unique entity address for each sensor, together with the sensor's latitude and longitude. -This "PostWeatherDataRequest" message posts temperature forecasts for 15-minute intervals between 3.00pm and 4.30pm for a weather sensor located at latitude 33.4843866 and longitude 126.477859. TODO: which is in Korea's timezone, see see iso datetimes. -The forecasts were made at noon. +This "PostWeatherDataRequest" message posts temperature forecasts for 15-minute intervals between 3.00pm and 4.30pm for a weather sensor located at latitude 33.4843866 and longitude 126.477859. This sensor is located in Korea's timezone ― we also reflect that in the datetimes. +The forecasts were made at noon, as the ``prior`` field indicates. .. code-block:: json @@ -56,6 +63,9 @@ The forecasts were made at noon. "unit": "°C" } +Note how the resolution of the data comes out at 15 minutes, if you divide the duration by the number of data points. If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match and if that is not possible, complain. + + Observations vs forecasts ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -70,6 +80,18 @@ For example, a horizon of "-PT1H" would denote that each temperature reading was See :ref:`prognoses` for more information regarding the prior and horizon fields. +Collecting weather data from OpenWeatherMap +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For convenience, we built in a CLI task which collects weather measurements and forecasts from the OpenWeatherMap API. You have to add your own token in the OPENWEATHERMAP_API_KEY setting first. Then you could run this task periodically, probably once per hour. Here is how: + +.. code-block:: + + flexmeasures add external-weather-forecasts --location 33.4366,126.5269 --store-in-db + +Consult the ``--help`` for this command to learn more about what you can do with it. + + Posting price data ------------------ @@ -82,7 +104,7 @@ Price data (both observations and forecasts) can be posted to the following POST This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction. The horizon indicates that the prices were published at 3pm on December 31st 2014 -(i.e. 33 hours ahead of midnight the next day). +(i.e. 33 hours ahead of midnight the next day which is the clearing time of KPX ― see below for a deeper explanation). .. code-block:: json @@ -150,6 +172,8 @@ while forecasts of power data can be posted to the following POST endpoint: For both endpoints, power data can be posted in various ways. The following examples assume that the endpoint for power data observations (i.e. meter data) is used. +.. todo:: For the time being, only one rate unit (MW) can be used to post power values. + Single value, single connection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -269,31 +293,8 @@ Multiple values (indicating a univariate timeseries) for 15-minute time interval "unit": "MW" } -Getting prognoses ------------------ - -Prognoses are power forecasts that are used by FlexMeasures to determine the best control signals to valorise on -balancing opportunities. Researchers can check the accuracy of these forecasts by downloading the prognoses and -comparing them against the meter data, i.e. the realised power measurements. -A prognosis can be requested for a single asset at the following GET endpoint: -.. code-block:: html - - https://company.flexmeasures.io/api//getPrognosis - -This example requests a prognosis with a rolling horizon of 6 hours before realisation. - -.. code-block:: json - - { - "type": "GetPrognosisRequest", - "connection": "ea1.2018-06.io.flexmeasures.company:1:1", - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT24H", - "horizon": "PT6H", - "resolution": "PT15M", - "unit": "MW" - } +.. _posting_flex_constraints: Posting flexibility constraints ------------------------------- @@ -335,66 +336,3 @@ As an example, consider the same UDI event as above with an additional target va } ] } - -Getting control signals ------------------------ - -A Prosumer can query FlexMeasures for control signals for its flexible devices using the following GET endpoint: - - -.. code-block:: html - - https://company.flexmeasures.io/api//getDeviceMessage - -Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. -This example requests a control signal for UDI event 203 posted previously. - -.. code-block:: json - - { - "type": "GetDeviceMessageRequest", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:203:soc" - } - -The following example response indicates that FlexMeasures planned ahead 45 minutes. -The list of consecutive power values represents the target consumption of the battery (negative values for production). -Each value represents the average power over a 15 minute time interval. - -.. sourcecode:: json - - { - "type": "GetDeviceMessageResponse", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:203", - "values": [ - 2.15, - 3, - 2 - ], - "start": "2015-06-02T10:00:00+00:00", - "duration": "PT45M", - "unit": "MW" - } - -One way of reaching the target consumption in this example is to let the battery start to consume with 2.15 MW at 10am, -increase its consumption to 3 MW at 10.15am and decrease its consumption to 2 MW at 10.30am. -However, because the targets values represent averages over 15-minute time intervals, the battery still has some degrees of freedom. -For example, the battery might start to consume with 2.1 MW at 10.00am and increase its consumption to 2.25 at 10.10am, -increase its consumption to 5 MW at 10.15am and decrease its consumption to 2 MW at 10.20am. -That should result in the same average values for each quarter-hour. - -Resetting the server --------------------- - -All power, price and weather data on the simulation server can be cleared using the following PUT endpoint (admin rights are required): - -.. code-block:: html - - https://company.flexmeasures.io/api//restoreData - -This example restores the database to a backup named demo_v0, which contains no timeseries data. - -.. code-block:: json - - { - "backup": "demo_v0" - } From 86ee0f3bf45cb99a63960dd4c2b9155595dfdbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 10 May 2021 13:31:03 +0200 Subject: [PATCH 06/11] finish forecasting&scheduling tutorial, finding an acceptable trade-off for explaining postUdiEvents; various smaller improvements --- documentation/concepts/algorithms.rst | 4 + documentation/getting-started.rst | 5 ++ documentation/index.rst | 1 + documentation/tut/forecasting_scheduling.rst | 79 +++++++++++++++++--- documentation/tut/posting_data.rst | 40 ++++------ flexmeasures/api/v1_3/routes.py | 2 +- 6 files changed, 94 insertions(+), 37 deletions(-) diff --git a/documentation/concepts/algorithms.rst b/documentation/concepts/algorithms.rst index aaadf1c5f..6221083c6 100644 --- a/documentation/concepts/algorithms.rst +++ b/documentation/concepts/algorithms.rst @@ -9,6 +9,8 @@ Algorithms :depth: 2 +.. _algorithms_forecasting: + Forecasting ----------- @@ -73,6 +75,8 @@ Improvements: - Most assets have yearly seasonality (e.g. wind, solar) and therefore forecasts would benefit from >= 2 years of history. +.. _algorithms_scheduling: + Scheduling ------------ diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 8d8634d7d..177cef895 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -211,3 +211,8 @@ Installing Cbc can be done on Unix via: We provide a script for installing from source (without requiring ``sudo`` rights) in :ref:`continuous_integration`. More information (e.g. for installing on Windows) on `the Cbc website `_. + +Install and configure Redis +^^^^^^^^^^^^^^^^^^^^^^^ + +To let FlexMeasures queue forecasting and scheduling jobs, install a `Redis `_ server (or rent one) and configure access to it within FlexMeasures' config file (see above). You can find the necessary settings in :ref:`redis-config`. \ No newline at end of file diff --git a/documentation/index.rst b/documentation/index.rst index 0f95993a9..1cb53a153 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -50,6 +50,7 @@ The platform operator of FlexMeasures can be an Aggregator. :maxdepth: 1 tut/posting_data + tut/forecasting_scheduling .. toctree:: :caption: The in-built UI diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 8f80aa462..e49cc2532 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -3,15 +3,17 @@ Forecasting & scheduling ======================== -Once FlexMeasures has been integrated with data (see :ref:`_tut_posting_data`), you can enjoy its forecasting and scheduling services. -Let's take a look how to set this up (if you are hosting FlexMeasures yourself) and how to access this information. +Once FlexMeasures has been integrated with data (see :ref:`tut_posting_data`), you can enjoy its forecasting and scheduling services. +Let's take a look how to access this information (for all users of FlexMeasures) and how to set the data science queues for this up (if you are hosting FlexMeasures yourself). -.. note: If you are not hosting FlexMeasures yourself, skip to :ref:`getting_prognoses`. +If you want to learn more about the actual algorithms used in the background, head over to :ref:`algorithms`. Maintaining the queues ------------------------------------ +.. note:: If you are not hosting FlexMeasures yourself, skip right ahead to :ref:`how_queue_forecasting` or :ref:`getting_prognoses`. + Here we assume you have access to a Redis server and configured it (see :ref:`redis-config`). Start to run one worker for each kind of job (in a separate terminal): @@ -35,13 +37,68 @@ When the main FlexMeasures process runs (e.g. by ``flexmeasures run``\ ), the qu When forecasts and schedules have been generated, they should be visible at ``http://localhost:5000/analytics``. -Queueing jobs +.. _how_queue_forecasting: + +How forecasting jobs are queued +------------------ + +A forecasting job is the order to create forecasts based on measurements. A job can be about forecasting one one point in time or about a range of points. + + +In FlexMeasures, forecasting jobs are created by the API when new power, weather or price data arrives (see :ref:`_tut_posting_data`). So technically, you don't have to do anything to keep fresh forecasts. + +The decision which horizons to forecast is currently also taken by FlexMeasures. For power data, FlexMeasures makes this decision depending on the asset resolution. For instance, a resolution of 15 minutes leads to forecast horizons of 1, 6, 24 and 48 hours. For price data, FlexMeasures chooses to forecast prices forward 24 and 48 hours +These are decent defaults, and fixing them has the advantage that scheduling scripts (see below) will know what to expect. However, horizons will probably become more configurable in the near future of FlexMeasures. + +Forecasting historic ranges +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There might be reasons to add forecasts of past time ranges. For instance, for visualisation of past system behaviour and to check how well the forecasting models have been doing on a longer stretch of data. + +If you host FlexMeasures yourself, we provide a CLI task for adding forecasts for whole historic periods. This is an example call: + +.. code-block:: bash + + flexmeasures add forecasts --from_date 2020-01-02 --to_date 2020-6-30 --horizon_hours 6 --asset-id 2 + +Here, forecasts are being computed for asset 2, with one horizon (6 hours). This is half year of data, so it will take a while. You can also queue this work to workers (see above) with the additional ``--as-job`` parameter (though in general we'd advise to dispatch this work in smaller chunks). + +.. _how_queue_scheduling: + +How scheduling jobs are queued ------------------ -TODO: Explain how forecasting jobs are made by the API when new data arrives. Scheduling jobs also, when PostUdiEvent is called. Future work: how can a user configure which assets get this treatment? +In FlexMeasures, a scheduling job is the order to plan optimized actions for flexible devices. It usually involves a linear program which draws on forecasted data so it can plan energy flexibility ahead of time. -TODO: Show that we have flexmeasures add forecasts, example: --from_date 2015-02-02 --to_date 2015-02-04 --horizon_hours 6 --asset-id 2 --as-job - Drawback: only for complete days (can we change that?) +We already learned about the ``postUdiEvent`` endpoint in :ref:`_posting_flex_states`, where we saw how to post a state of flexibility (in this case, the state of charge of a battery at a certain point in time). + +This endpoint can also be used to request a future state if charge (using ``soc-with-target`` in the entity address). +"battery", "one-way_evse", "two-way_evse" +"soc", "soc-with-targets" + +As an example, consider the same UDI event as we saw earlier, but with an additional target value. + +.. code-block:: json + + { + "type": "PostUdiEventRequest", + "event": "ea1.2018-06.io.flexmeasures.company:7:10:204:soc-with-targets", + "value": 12.1, + "datetime": "2015-06-02T10:00:00+00:00", + "unit": "kWh", + "targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + } + ] + } + +Here we have described the state of charge at 10am to be ``12.1``. In addition, we requested that it should be ``25`` at 4pm. For instance, this could mean that a cat should be charged at 90% at that time. + +Now, here is a task that requires some scheduling. If FlexMeasures receives this UDI Event, a scheduling job will be made and put into the queue. In turn, the forecasting job creates a proposed schedule. We'll look a bit deeper into those further down in :ref:`getting_schedules`; + +.. note:: Even without a target state of charge, FlexMeasures will create a scheduling job. The flexible device can then be used with more freedom to reach the system objective (e.g. store power when it is cheap, sell when it's expensive). .. _getting_prognoses: @@ -52,7 +109,7 @@ Getting forecasts (prognoses) Prognoses (the USEF term used for forecasts) are used by FlexMeasures to determine the best control signals to valorise on balancing opportunities. -You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis `_ +You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. Getting them might be useful if you want to use prognoses in your own system or to check the accuracy of these forecasts by downloading the prognoses and comparing them against the meter data, i.e. the realised power measurements (though the FlexMeasures UI also visualises them next to each other). @@ -77,17 +134,19 @@ This example requests a prognosis for 24 hours, with a rolling horizon of 6 hour } +.. _getting_schedules: + Getting schedules (control signals) ----------------------- -FlexMeasures can create optimised schedules with control signals for flexible devices. You can access the schedules via the `GET /api/v2_0/getDeviceMessage `_ endpoint. The URL then looks like this: +We saw above how FlexMeasures can create optimised schedules with control signals for flexible devices. You can access the schedules via the `GET /api/v2_0/getDeviceMessage <../api/v2_0.html#get--api-v2_0-getDeviceMessage>`_ endpoint. The URL then looks like this: .. code-block:: html https://company.flexmeasures.io/api//getDeviceMessage Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. -This example of a request body shows that we want to look up a control signal for UDI event 203 (which was posted previously, see :ref:`posting_flex_constraints`). +This example of a request body shows that we want to look up a control signal for UDI event 203 (which was posted previously, see :ref:`posting_flex_states`). .. code-block:: json diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 9924c7b69..f2bc0e845 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -27,7 +27,7 @@ Prerequisites Posting weather data -------------------- -Weather data (both observations and forecasts) can be posted to the following POST endpoint: +Weather data (both observations and forecasts) can be posted to `POST /api/v2_0/postWeatherData <../api/v2_0.html#post--api-v2_0-postWeatherData>`_. The URL might look like this: .. code-block:: html @@ -83,7 +83,7 @@ See :ref:`prognoses` for more information regarding the prior and horizon fields Collecting weather data from OpenWeatherMap ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For convenience, we built in a CLI task which collects weather measurements and forecasts from the OpenWeatherMap API. You have to add your own token in the OPENWEATHERMAP_API_KEY setting first. Then you could run this task periodically, probably once per hour. Here is how: +For convenience for people who host FlexMeasures themselves, we built in a CLI task which collects weather measurements and forecasts from the OpenWeatherMap API. You have to add your own token in the OPENWEATHERMAP_API_KEY setting first. Then you could run this task periodically, probably once per hour. Here is how: .. code-block:: @@ -95,7 +95,7 @@ Consult the ``--help`` for this command to learn more about what you can do with Posting price data ------------------ -Price data (both observations and forecasts) can be posted to the following POST endpoint: +Price data (both observations and forecasts) can be posted to `POST /api/v2_0/postPriceData <../api/v2_0.html#post--api-v2_0-postPriceData>`_. The URL might look like this: .. code-block:: html @@ -157,13 +157,13 @@ Posting power data For power data, USEF specifies separate message types for observations and forecasts. Correspondingly, FlexMeasures uses separate endpoints to communicate these messages. -Observations of power data can be posted to the following POST endpoint: +Observations of power data can be posted to `POST /api/v2_0/postMeterData <../api/v2_0.html#post--api-v2_0-postMeterData>`_. The URL might look like this: .. code-block:: html https://company.flexmeasures.io/api//postMeterData -while forecasts of power data can be posted to the following POST endpoint: +while forecasts of power data can be posted to `POST /api/v2_0/postPrognosis <../api/v2_0.html#post--api-v2_0-postPrognosis>`_. The URL might look like this: .. code-block:: html @@ -294,19 +294,22 @@ Multiple values (indicating a univariate timeseries) for 15-minute time interval } -.. _posting_flex_constraints: +.. _posting_flex_states: -Posting flexibility constraints +Posting flexibility states ------------------------------- -Prosumers that have Active Demand & Supply can post the constraints of their flexible devices to FlexMeasures at the -following POST endpoint: +There is one more crucial kind of data that FlexMeasures needs to know about: What are the current states of flexible devices? For example, a battery has a state of charge. + +The USEF framework defines a so-called "UDI-Event" (UDI stands for Universal Device Interface) to communicate settings for devices with Active Demand & Supply (ADS). +Owners of such devices can post these states to `POST /api/v2_0/postUdiEvent <../api/v2_0.html#post--api-v2_0-postUdiEvent>`_. The URL might look like this: .. code-block:: html https://company.flexmeasures.io/api//postUdiEvent This example posts a state of charge value for a battery device (asset 10 of owner 7) as UDI event 203. +This way, FlexMeasures knows how much of the potential flexible energy services this battery can provide in the near future. .. code-block:: json @@ -318,21 +321,6 @@ This example posts a state of charge value for a battery device (asset 10 of own "unit": "kWh" } -Some devices also accept target values for their state of charge. -As an example, consider the same UDI event as above with an additional target value. - -.. code-block:: json +.. note:: At the moment, FlexMeasures only supports batteries and car chargers here (asset types "battery", "one-way_evse" or "two-way_evse"), but this will be expanded to flexible assets as needed. - { - "type": "PostUdiEventRequest", - "event": "ea1.2018-06.io.flexmeasures.company:7:10:204:soc-with-targets", - "value": 12.1, - "datetime": "2015-06-02T10:00:00+00:00", - "unit": "kWh", - "targets": [ - { - "value": 25, - "datetime": "2015-06-02T16:00:00+00:00" - } - ] - } +Actually, UDI Events are more powerful than this. In :ref:`how_queue_scheduling`, we'll cover how they can be used to request a future state, which is useful to steer the scheduling. \ No newline at end of file diff --git a/flexmeasures/api/v1_3/routes.py b/flexmeasures/api/v1_3/routes.py index cedcb7092..0c02c3220 100644 --- a/flexmeasures/api/v1_3/routes.py +++ b/flexmeasures/api/v1_3/routes.py @@ -80,7 +80,7 @@ def get_device_message(): @auth_token_required @usef_roles_accepted(*list_access(v1_3_service_listing, "postUdiEvent")) def post_udi_event(): - """API endpoint to post UDI event. + """API endpoint to post UDI event. (UDI is the Universal Device Interface proposed by USEF for flexible device states) .. :quickref: Control; Upload flexibility constraints to the platform From 7ea0c8ead7faa3e883fc4d5f9dfd283fc5b44da3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 10 May 2021 16:09:35 +0200 Subject: [PATCH 07/11] tutorial about buiding custom UIs --- documentation/index.rst | 1 + documentation/tut/building-uis.rst | 10 - documentation/tut/building_uis.rst | 284 +++++++++++++++++++++++++++++ 3 files changed, 285 insertions(+), 10 deletions(-) delete mode 100644 documentation/tut/building-uis.rst create mode 100644 documentation/tut/building_uis.rst diff --git a/documentation/index.rst b/documentation/index.rst index 1cb53a153..782611185 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -51,6 +51,7 @@ The platform operator of FlexMeasures can be an Aggregator. tut/posting_data tut/forecasting_scheduling + tut/building_uis .. toctree:: :caption: The in-built UI diff --git a/documentation/tut/building-uis.rst b/documentation/tut/building-uis.rst deleted file mode 100644 index f0d0f0b33..000000000 --- a/documentation/tut/building-uis.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _tut_building_uis: - -Building UIs on top of the FlexMeasures API -======================== - -TODO: Explain why using FlexMeasure information in custom UIs might be beneficial. - -TODO: show three calls to our API which we didn't show yet: user info, asset info and graphs. Example with generic JavaScript how to embed in a HTML page. - - diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst new file mode 100644 index 000000000..19470daea --- /dev/null +++ b/documentation/tut/building_uis.rst @@ -0,0 +1,284 @@ +.. _tut_building_uis: + +Building custom UIs +======================== + +FlexMeasures provides its own UI (see :ref:`dashboard`), but it is a back office platform first. Every energy service company who might use FlexMeasures usually already has their own user-facing system, as well. Thus, it might be highly useful to be able to incorporate information from FlexMeasures in custom UIs. + +This tutorial will show how the FlexMeasures API can be used from JavaScript to extract information and display it in a browser (using HTML). We'll extract information about users, assets and even whole plots! + +.. contents:: + :local: + :depth: 1 + + +.. note:: We'll use standard JavaScript for this tutorial, in particular the `fetch `_ functionality, which many browsers support out-of-the-box these days. You might want to use more high-level frameworks like jQuery, Angular, React or VueJS for your frontend, of course. + + +Get an authentication token +----------------------- + +FlexMeasures provides the `POST /api/v2_0/requestAuthToken <../api/v2_0.html#post--api-v2_0-requestAuthToken>`_ endpoint, as discussed in :ref:`api_auth`. +Here is a JavaScript function to call it: + +.. code-block:: JavaScript + + var flexmeasures_domain = "http://localhost:5000"; + + function getAuthToken(){ + return fetch(flexmeasures_domain + '/api/requestAuthToken', + { + method: "POST", + mode: "cors", + headers: + { + "Content-Type": "application/json", + }, + body: JSON.stringify({"email": email, "password": password}) + } + ) + .then(function(response) { return response.json(); }) + .then(console.log("Got auth token from FlexMeasures server ...")); + } + +It only expects you to set ``email`` and ``password`` somewhere (you could also pass them in). In addition, we expect here that ``flexmeasures_domain`` is set to the FlexMeasures server you interact with, for example "https://company.flexmeasures.io". + +We'll see how to make use of the ``getAuthToken`` function right away, keep on reading. + + + + +Load user information +----------------------- + +Let's say we are interested in a particular user's meta data. For instance, which email address do they have and which timezone are they operating in? + +Here is some code to find out and display that information in a simple HTML table: + + +.. code-block:: html + +

User info

+

+ Email address: +

+

+ Time zone: +

+ +.. code-block:: JavaScript + + function loadUserInfo(userId, authToken) { + fetch(flexmeasures_domain + '/api/v2_0/user/' + userId, + { + method: "GET", + mode: "cors", + headers: + { + "Content-Type": "application/json", + "Authorization": authToken + }, + } + ) + .then(console.log("Got user data from FlexMeasures server ...")) + .then(function(response) { return response.json(); }) + .then(function(userInfo) { + document.querySelector('#user_email').innerHTML = userInfo.email; + document.querySelector('#user_timezone').innerHTML = userInfo.timezone; + }) + } + + document.onreadystatechange = () => { + if (document.readyState === 'complete') { + getAuthToken() + .then(function(response) { + var authToken = response.auth_token; + loadUserInfo(userId, authToken); + }) + } + } + +The result looks like this in your browser: + +.. image:: https://github.com/SeitaBV/screenshots/raw/main/tut/user_info.png + :align: center +.. :scale: 40% + + +From FlexMeasures, we are using the `GET /api/v2_0/user <../api/v2_0.html#get--api-v2_0-user-(id)>`_ endpoint, which loads information about one user. Browse its documentation to learn other information you could get. + + +Load asset information +----------------------- + +Similarly, we can load asset information. Say we have a user ID and we want to show which assets FlexMeasures administrates for that user. + + +.. code-block:: html + + + + + + + + + + +
Asset nameTypeCapacity
+ + +.. code-block:: JavaScript + + function loadAssets(userId, authToken) { + var params = new URLSearchParams(); + params.append("owner_id", userId); + fetch(flexmeasures_domain + '/api/v2_0/assets?' + params.toString(), + { + method: "GET", + mode: "cors", + headers: + { + "Content-Type": "application/json", + "Authorization": authToken + }, + } + ) + .then(console.log("Got asset data from FlexMeasures server ...")) + .then(function(response) { return response.json(); }) + .then(function(rows) { + rows.forEach(row => { + const tbody = document.querySelector('#assetTable tbody'); + const tr = document.createElement('tr'); + tr.innerHTML = `${row.display_name}${row.asset_type_name}${row.capacity_in_mw} MW`; + tbody.appendChild(tr); + }); + }) + } + + document.onreadystatechange = () => { + if (document.readyState === 'complete') { + getAuthToken() + .then(function(response) { + var authToken = response.auth_token; + loadAssets(userId, authToken); + }) + } + } + + +The result looks like this in your browser: + +.. image:: https://github.com/SeitaBV/screenshots/raw/main/tut/asset_info.png + :align: center +.. :scale: 40% + + + +From FlexMeasures, we are using the `GET /api/v2_0/assets <../api/v2_0.html#get--api-v2_0-assets>`_ endpoint, which loads a list of assets. Note how, unlike the user endpoint above, we are passing a query parameter here (``owner_id``). We are only displaying a subset of the information which is available about assets. Browse the endpoint documentation to learn other information you could get. + + +Embedding plots +------------------------ + +Creating plots from data can consume lots of development time. FlexMeasures can help here by delivering ready-made plots. + +In this tutorial, let's display two plots: one with power measurements and forecasts (a solar panel installation) and one with schedules of several EV chargers on the same location, next to each other for easy comparison. + +First, we define two div tags for the two plots and a basic layout for them. We also load the Bokeh library, more about that below. + +.. code-block:: html + + + +.. code-block:: html + + +
+
+
+
+ +Now we define a JavaScript function to ask the FlexMeasures API for a plot: + +.. code-block:: JavaScript + + function renderPlot(params, authToken, divId){ + fetch(flexmeasures_domain + '/api/v2_0/charts/power?' + params.toString(), + { + method: "GET", + mode: "cors", + headers: + { + "Content-Type": "application/json", + "Authorization": authToken + }, + } + ) + .then(function(response) { return response.json(); }) + .then(function(item) { Bokeh.embed.embed_item(item, divId); }) + .then(console.log("Got plot specifications from server and rendered it ...")) + } + +This function allows us to request a plot (actually, HTML and JavaScript code to render a plot), and then render the plot within a ``div`` tag of our choice. + +As FlexMeasures uses `the Bokeh Visualization Library `_ internally, we also need to import the Bokeh client library to render the plots (see the ``script`` tag above). It's crucial to note that FlexMeasures is not transferring images across HTTP here, just information needed to render them. + +.. note:: The Bokeh library version you use in your frontend needs to match the version which FlexMeasures uses internally, check ``requirements/app.txt`` when in doubt. + +Now let's call this function when the HTML page is opened, to load our two plots: + +.. code-block:: JavaScript + + document.onreadystatechange = () => { + if (document.readyState === 'complete') { + getAuthToken() + .then(function(response) { + var authToken = response.auth_token; + + var urlData1 = new URLSearchParams(); + urlData1.append("resource", "ss_pv"); + urlData1.append("start_time", "2015-06-01T10:00:00"); + urlData1.append("end_time", "2015-06-03T10:00:00"); + urlData1.append("resolution", "PT15M"); + urlData1.append("forecast_horizon", "PT6H"); + urlData1.append("show_individual_traces_for", "none"); + renderPlot(urlData1, authToken, "plot-div1"); + + var urlData2 = new URLSearchParams(); + urlData2.append("resource", "Test station (Charge Point)"); + urlData2.append("start_time", "2015-01-01T00:00:00"); + urlData2.append("end_time", "2015-01-01T03:00:00"); + urlData2.append("resolution", "PT15M"); + urlData2.append("show_individual_traces_for", "schedules"); + renderPlot(urlData2, authToken, "plot-div2"); + }) + } + } + +For each of the two plots we request, we pass in several query parameters to describe what we want to see. We define which asset and what time range, which resolution and forecasting horizon. +Note the ``show_individual_traces_for`` setting - it allows us to split data from individual assets (usually measurements, forecasts and schedules are visually aggregated in FlexMeasure's power plots, see :ref:`analytics` for example). + + +The result looks like this in your browser: + +.. image:: https://github.com/SeitaBV/screenshots/raw/main/tut/plots.png + :align: center +.. :scale: 40% + + +From FlexMeasures, we are using the `GET /api/v2_0/charts/power <../api/v2_0.html#get--api-v2_0-charts-power>`_ endpoint, which loads HTML and JavaScript. +Browse the endpoint documentation to learn more about it. \ No newline at end of file From 184d975f46b13ed29d07c096297cf634c60ae44a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 10 May 2021 16:23:52 +0200 Subject: [PATCH 08/11] fix note --- documentation/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 177cef895..c6cff021e 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -8,7 +8,7 @@ Quickstart This section walks you through getting FlexMeasures to run with the least effort. We'll cover making a secret key, connecting a database and creating one user & one asset. -.. note: Are you not hosting FlexMeasures, but wanting to learn how to use it? head over to our tutorials, starting with :ref:`_tut_posting_data`. +.. note:: Are you not hosting FlexMeasures, but wanting to learn how to use it? Head over to our tutorials, starting with :ref:`tut_posting_data`. Install FlexMeasures From b0195fad8d3bc32d4c11aec8c45ab397cc17d24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 10 May 2021 16:29:33 +0200 Subject: [PATCH 09/11] use company.flexmeasures.io as example URL in API introduction throughout --- documentation/api/introduction.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index bf484ee49..3420463a1 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -4,7 +4,6 @@ Introduction ============ This document details the Application Programming Interface (API) of the FlexMeasures web service. The API supports user automation for flexibility valorisation in the energy sector, both in a live setting and for the purpose of simulating scenarios. The web service adheres to the concepts and terminology used in the Universal Smart Energy Framework (USEF). -We assume in this document that the FlexMeasures instance you want to connect to is hosted at https://company.flexmeasures.io. .. _api_versions: @@ -32,23 +31,29 @@ At Seita, we run servers for our clients at: where `company` is a hosting customer of ours. All their accounts' data lives on that server. -Let's see what it says: +We assume in this document that the FlexMeasures instance you want to connect to is hosted at https://company.flexmeasures.io. + +Let's see what the ``/api`` endpoint returns: .. code-block:: python >>> import requests - >>> res = requests.get("https://seita.flexmeasures.io/api") + >>> res = requests.get("https://company.flexmeasures.io/api") >>> res.json() - {'flexmeasures_version': '0.4.0', 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: /api/requestAuthToken', 'status': 200, 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0']} + {'flexmeasures_version': '0.4.0', + 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: /api/requestAuthToken', + 'status': 200, + 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0'] + } So this tells us which API versions exist. For instance, we know that the latest API version is available at .. code-block:: html - https://seita.flexmeasures.io/api/v2_0 + https://company.flexmeasures.io/api/v2_0 -Also , we see that a list of services offered by (a version of) the FlexMeasures web service can be obtained by sending a *getService* request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. +Also , we see that a list of services offered by (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. **Example request** @@ -56,7 +61,7 @@ Let's ask which endpoints are available for meter data companies (MDC): .. code-block:: html - https://seita.flexmeasures.io/api/v2_0/getService?access=MDC + https://company.flexmeasures.io/api/v2_0/getService?access=MDC **Example response** From 51fb336962e6bdf1f6c49d09efcda0997b2c3183 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 18 May 2021 12:54:25 +0200 Subject: [PATCH 10/11] Improve linguistic flow --- documentation/tut/building_uis.rst | 9 ++-- documentation/tut/forecasting_scheduling.rst | 51 +++++++++++--------- documentation/tut/posting_data.rst | 23 +++++---- 3 files changed, 49 insertions(+), 34 deletions(-) diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst index 19470daea..bb9baacc6 100644 --- a/documentation/tut/building_uis.rst +++ b/documentation/tut/building_uis.rst @@ -3,11 +3,13 @@ Building custom UIs ======================== -FlexMeasures provides its own UI (see :ref:`dashboard`), but it is a back office platform first. Every energy service company who might use FlexMeasures usually already has their own user-facing system, as well. Thus, it might be highly useful to be able to incorporate information from FlexMeasures in custom UIs. +FlexMeasures provides its own UI (see :ref:`dashboard`), but it is a back office platform first. +Most energy service companies already have with their own user-facing system. +We therefore made it possible to incorporate information from FlexMeasures in custom UIs. This tutorial will show how the FlexMeasures API can be used from JavaScript to extract information and display it in a browser (using HTML). We'll extract information about users, assets and even whole plots! -.. contents:: +.. contents:: Table of contents :local: :depth: 1 @@ -105,7 +107,8 @@ The result looks like this in your browser: .. :scale: 40% -From FlexMeasures, we are using the `GET /api/v2_0/user <../api/v2_0.html#get--api-v2_0-user-(id)>`_ endpoint, which loads information about one user. Browse its documentation to learn other information you could get. +From FlexMeasures, we are using the `GET /api/v2_0/user <../api/v2_0.html#get--api-v2_0-user-(id)>`_ endpoint, which loads information about one user. +Browse its documentation to learn about other information you could get. Load asset information diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index e49cc2532..c9bb30cc3 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -3,8 +3,12 @@ Forecasting & scheduling ======================== -Once FlexMeasures has been integrated with data (see :ref:`tut_posting_data`), you can enjoy its forecasting and scheduling services. -Let's take a look how to access this information (for all users of FlexMeasures) and how to set the data science queues for this up (if you are hosting FlexMeasures yourself). +Once FlexMeasures contains data (see :ref:`tut_posting_data`), you can enjoy its forecasting and scheduling services. +Let's take a look at how FlexMeasures users can access information from these services, and how you (if you are hosting FlexMeasures yourself) can set up the data science queues for this. + +.. contents:: Table of contents + :local: + :depth: 1 If you want to learn more about the actual algorithms used in the background, head over to :ref:`algorithms`. @@ -42,15 +46,17 @@ When forecasts and schedules have been generated, they should be visible at ``ht How forecasting jobs are queued ------------------ -A forecasting job is the order to create forecasts based on measurements. A job can be about forecasting one one point in time or about a range of points. +A forecasting job is an order to create forecasts based on measurements. +A job can be about forecasting one point in time or about forecasting a range of points. -In FlexMeasures, forecasting jobs are created by the API when new power, weather or price data arrives (see :ref:`_tut_posting_data`). So technically, you don't have to do anything to keep fresh forecasts. +In FlexMeasures, forecasting jobs are created by the server when new power, weather or price data arrives through the API (see :ref:`tut_posting_data`). +So technically, you don't have to do anything to keep fresh forecasts. The decision which horizons to forecast is currently also taken by FlexMeasures. For power data, FlexMeasures makes this decision depending on the asset resolution. For instance, a resolution of 15 minutes leads to forecast horizons of 1, 6, 24 and 48 hours. For price data, FlexMeasures chooses to forecast prices forward 24 and 48 hours These are decent defaults, and fixing them has the advantage that scheduling scripts (see below) will know what to expect. However, horizons will probably become more configurable in the near future of FlexMeasures. -Forecasting historic ranges +Historical forecasts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There might be reasons to add forecasts of past time ranges. For instance, for visualisation of past system behaviour and to check how well the forecasting models have been doing on a longer stretch of data. @@ -61,22 +67,23 @@ If you host FlexMeasures yourself, we provide a CLI task for adding forecasts fo flexmeasures add forecasts --from_date 2020-01-02 --to_date 2020-6-30 --horizon_hours 6 --asset-id 2 -Here, forecasts are being computed for asset 2, with one horizon (6 hours). This is half year of data, so it will take a while. You can also queue this work to workers (see above) with the additional ``--as-job`` parameter (though in general we'd advise to dispatch this work in smaller chunks). +Here, forecasts are being computed for asset 2, with one horizon (6 hours). +This is half a year of data, so it will take a while. +You can also queue this work to workers (see above) with the additional ``--as-job`` parameter (though in general we'd advise to dispatch this work in smaller chunks). .. _how_queue_scheduling: How scheduling jobs are queued ------------------ -In FlexMeasures, a scheduling job is the order to plan optimized actions for flexible devices. It usually involves a linear program which draws on forecasted data so it can plan energy flexibility ahead of time. +In FlexMeasures, a scheduling job is an order to plan optimised actions for flexible devices. +It usually involves a linear program that combines a state of energy flexibility with forecasted data to draw up a consumption or production plan ahead of time. -We already learned about the ``postUdiEvent`` endpoint in :ref:`_posting_flex_states`, where we saw how to post a state of flexibility (in this case, the state of charge of a battery at a certain point in time). +We already learned about the ``postUdiEvent`` endpoint in :ref:`posting_flex_states`, where we saw how to post a state of flexibility (in this case, the state of charge of a battery at a certain point in time). -This endpoint can also be used to request a future state if charge (using ``soc-with-target`` in the entity address). -"battery", "one-way_evse", "two-way_evse" -"soc", "soc-with-targets" +This endpoint can also be used to request a future state of charge (using ``soc-with-target`` in the entity address). -As an example, consider the same UDI event as we saw earlier, but with an additional target value. +As an example, consider the same UDI event as we saw earlier (in :ref:`posting_flex_states`), but with an additional target value. .. code-block:: json @@ -94,26 +101,26 @@ As an example, consider the same UDI event as we saw earlier, but with an additi ] } -Here we have described the state of charge at 10am to be ``12.1``. In addition, we requested that it should be ``25`` at 4pm. For instance, this could mean that a cat should be charged at 90% at that time. +Here we have described the state of charge at 10am to be ``12.1``. In addition, we requested that it should be ``25`` at 4pm. +For instance, this could mean that a car should be charged at 90% at that time. -Now, here is a task that requires some scheduling. If FlexMeasures receives this UDI Event, a scheduling job will be made and put into the queue. In turn, the forecasting job creates a proposed schedule. We'll look a bit deeper into those further down in :ref:`getting_schedules`; +Now here is a task that requires some scheduling. If FlexMeasures receives this UDI Event, a scheduling job will be made and put into the queue. In turn, the forecasting job creates a proposed schedule. We'll look a bit deeper into those further down in :ref:`getting_schedules`; -.. note:: Even without a target state of charge, FlexMeasures will create a scheduling job. The flexible device can then be used with more freedom to reach the system objective (e.g. store power when it is cheap, sell when it's expensive). +.. note:: Even without a target state of charge, FlexMeasures will create a scheduling job. The flexible device can then be used with more freedom to reach the system objective (e.g. buy power when it is cheap, store it, and sell back when it's expensive). .. _getting_prognoses: -Getting forecasts (prognoses) +Getting power forecasts (prognoses) ----------------- -Prognoses (the USEF term used for forecasts) are used by FlexMeasures to determine the best control signals to valorise on -balancing opportunities. +Prognoses (the USEF term used for power forecasts) are used by FlexMeasures to determine the best control signals to valorise on balancing opportunities. You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. -Getting them might be useful if you want to use prognoses in your own system or to check the accuracy of these forecasts by downloading the prognoses and -comparing them against the meter data, i.e. the realised power measurements (though the FlexMeasures UI also visualises them next to each other). +Getting them might be useful if you want to use prognoses in your own system, or to check their accuracy against meter data, i.e. the realised power measurements. +The FlexMeasures UI also lists forecast accuracy, and visualises prognoses and meter data next to each other. -So a prognosis can be requested for a single asset at the ``getPrognosis`` endpoint, at an URL looking like this: +A prognosis can be requested for a single asset at the ``getPrognosis`` endpoint, at a URL looking like this: .. code-block:: html @@ -145,7 +152,7 @@ We saw above how FlexMeasures can create optimised schedules with control signal https://company.flexmeasures.io/api//getDeviceMessage -Control signals can be queried by UDI event for up to 1 week after the UDI event was posted. +Control signals can be queried by UDI event for up to 1 week after the UDI event was posted (ask your host if you need to keep them around longer). This example of a request body shows that we want to look up a control signal for UDI event 203 (which was posted previously, see :ref:`posting_flex_states`). .. code-block:: json diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index f2bc0e845..729a72e30 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -5,9 +5,11 @@ Posting data The platform FlexMeasures strives on the data you feed it. Let's demonstrate how you can get data into FlexMeasures using the API. This is where FlexMeasures gets connected to your system as a smart backend and helps you build smart energy services. -We will show how to use the API endpoints for POSTing data. You can call these in regular intervals (through scheduled scripts in your system, for example), so that FlexMeasures is always integrated with the recent data. Of course, these endpoints can also be used to load historic data into FlexMeasures, so that the forecasting models have enough data history to work with. +We will show how to use the API endpoints for POSTing data. +You can call these at regular intervals (through scheduled scripts in your system, for example), so that FlexMeasures always has recent data to work with. +Of course, these endpoints can also be used to load historic data into FlexMeasures, so that the forecasting models have access to enough data history. -.. note:: For the purposes of forecasting and scheduling, it is often advisable to use a higher resolution than most metering services keep. For example, while such services might measure every ten seconds, FlexMeasures will usually do its job no less effective if you feed it data with a resolution of five minutes. This will also make the data integration much easier. Keep also in mind that many data sources like weather forecasting or markets can have data resolutions of an hour, anyway. +.. note:: For the purposes of forecasting and scheduling, it is often advisable to use a less fine-grained resolution than most metering services keep. For example, while such services might measure every ten seconds, FlexMeasures will usually do its job no less effective if you feed it data with a resolution of five minutes. This will also make the data integration much easier. Keep in mind that many data sources like weather forecasting or markets can have data resolutions of an hour, anyway. .. contents:: Table of contents :local: @@ -63,7 +65,8 @@ The forecasts were made at noon, as the ``prior`` field indicates. "unit": "°C" } -Note how the resolution of the data comes out at 15 minutes, if you divide the duration by the number of data points. If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match and if that is not possible, complain. +Note how the resolution of the data comes out at 15 minutes when you divide the duration by the number of data points. +If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match or, if that is not possible, complain. Observations vs forecasts @@ -75,7 +78,7 @@ This denotes that the observation was made exactly after realisation of this lis Alternatively, to indicate that each individual observation was made directly after the end of its 15-minute interval (i.e. at 3.15pm, 3.30pm and so on), set a horizon to "PT0H" instead of a prior. Finally, delays in reading out sensor data can be simulated by setting the horizon field to a negative value. -For example, a horizon of "-PT1H" would denote that each temperature reading was observed one hour after the fact (i.e. at 4.15pm, 4.30 pm and so on). +For example, a horizon of "-PT1H" would denote that each temperature reading was observed one hour after the fact (i.e. at 4.15pm, 4.30pm and so on). See :ref:`prognoses` for more information regarding the prior and horizon fields. @@ -83,7 +86,8 @@ See :ref:`prognoses` for more information regarding the prior and horizon fields Collecting weather data from OpenWeatherMap ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For convenience for people who host FlexMeasures themselves, we built in a CLI task which collects weather measurements and forecasts from the OpenWeatherMap API. You have to add your own token in the OPENWEATHERMAP_API_KEY setting first. Then you could run this task periodically, probably once per hour. Here is how: +For convenience for organisations who host FlexMeasures themselves, we built in a CLI task which collects weather measurements and forecasts from the OpenWeatherMap API. +You have to add your own token in the OPENWEATHERMAP_API_KEY setting first. Then you could run this task periodically, probably once per hour. Here is how: .. code-block:: @@ -195,7 +199,7 @@ A single average power value for a 15-minute time interval for a single connecti Multiple values, single connection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after realisation. +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after each realisation. .. code-block:: json @@ -263,7 +267,7 @@ Single different values for a 15-minute time interval for two connections, poste Multiple values, multiple connections ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after realisation. +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after each realisation. .. code-block:: json @@ -309,7 +313,7 @@ Owners of such devices can post these states to `POST /api/v2_0/postUdiEvent <.. https://company.flexmeasures.io/api//postUdiEvent This example posts a state of charge value for a battery device (asset 10 of owner 7) as UDI event 203. -This way, FlexMeasures knows how much of the potential flexible energy services this battery can provide in the near future. +From this, FlexMeasures derives the energy flexibility this battery has in the near future. .. code-block:: json @@ -321,6 +325,7 @@ This way, FlexMeasures knows how much of the potential flexible energy services "unit": "kWh" } -.. note:: At the moment, FlexMeasures only supports batteries and car chargers here (asset types "battery", "one-way_evse" or "two-way_evse"), but this will be expanded to flexible assets as needed. +.. note:: At the moment, FlexMeasures only supports batteries and car chargers here (asset types "battery", "one-way_evse" or "two-way_evse"). + This will be expanded to flexible assets as needed. Actually, UDI Events are more powerful than this. In :ref:`how_queue_scheduling`, we'll cover how they can be used to request a future state, which is useful to steer the scheduling. \ No newline at end of file From 1744303a71b31f619c0800636544ba0aa71ad154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 21 May 2021 14:32:37 +0200 Subject: [PATCH 11/11] improve based on review feedback --- Makefile | 2 +- documentation/api/introduction.rst | 70 ++++++++++++++++++------------ documentation/getting-started.rst | 2 +- documentation/tut/posting_data.rst | 7 ++- 4 files changed, 47 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index dd8797f93..ce465bc92 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test: # ---- Documentation --- update-docs: - pip3 install sphinx sphinx-rtd-theme sphinxcontrib.httpdomain + pip3 install sphinx==3.5.4 sphinxcontrib.httpdomain # sphinx4 is not supported yet by sphinx-contrib/httpdomain, see https://github.com/sphinx-contrib/httpdomain/issues/46 cd documentation; make clean; make html; cd .. update-docs-pdf: diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 3420463a1..938541808 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -53,7 +53,7 @@ So this tells us which API versions exist. For instance, we know that the latest https://company.flexmeasures.io/api/v2_0 -Also , we see that a list of services offered by (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. +Also, we can see that a list of endpoints which are available at (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. **Example request** @@ -307,9 +307,11 @@ In case of a single group of connections, the message may be flattened to: Timeseries ^^^^^^^^^^ -Timestamps and durations are consistent with the ISO 8601 standard. The resolution of the data is implicit, see :ref:`resolutions`. +Timestamps and durations are consistent with the ISO 8601 standard. The resolution of the data is implicit (from duration and number of values), see :ref:`resolutions`. -All timestamps in requests to the API must be timezone-aware. The timezone indication "Z" indicates a zero offset from UTC. Additionally, we use the following shorthand for sequential values within a time interval: +All timestamps in requests to the API must be timezone-aware. For instance, in the below example, the timezone indication "Z" indicates a zero offset from UTC. + +We use the following shorthand for sending sequential, equidistant values within a time interval: .. code-block:: json @@ -323,7 +325,7 @@ All timestamps in requests to the API must be timezone-aware. The timezone indic "duration": "PT45M" } -is equal to: +Technically, this is equal to: .. code-block:: json @@ -352,31 +354,40 @@ This intuitive convention allows us to reduce communication by sending univariat Notation for v1 """"""""""""""" -For version 1 of the API, only univariate timeseries data is expected to be communicated. Therefore: +For version 1 and 2 of the API, only equidistant timeseries data is expected to be communicated. Therefore: + +- only the array notation should be used (first notation from above), +- "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and +- "duration" should also be a multiple of the sensor resolution. -- only the array notation should be used, -- "start" should be a timestamp on the hour or a multiple of 15 minutes thereafter, and -- "duration" should be a multiple of 15 minutes. .. _beliefs: -Beliefs -^^^^^^^ +Tracking the recording time of beliefs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For all its time series data, FlexMeasures keeps track of the time they were recorded. Data can be defined and filtered accordingly, which allows you to get a snapshot of what was known at a certain point in time. + +.. note:: FlexMeasures uses the `timely-beliefs data model `_ for modelling such facts about time series data, and accordingly we use the term "belief" in this documentation. In that model, the recording time is referred to as "belief time". + + +Querying by recording time +"""""""""""""""""""""""""""" -By regarding all time series data as beliefs that have been recorded at a certain time, data can be filtered accordingly. Some GET endpoints have two optional timing fields to allow such filtering. -The "prior" field (a timestamp) can be used to select beliefs recorded before some moment in time. + +The ``prior`` field (a timestamp) can be used to select beliefs recorded before some moment in time. It can be used to "time-travel" to see the state of information at some moment in the past. -In addition, the "horizon" field (a duration) can be used to select beliefs recorded before some moment in time, relative to each event. + +In addition, the ``horizon`` field (a duration) can be used to select beliefs recorded before some moment in time, `relative to each event`. For example, to filter out meter readings communicated within a day (denoted by a negative horizon) or forecasts created at least a day beforehand (denoted by a positive horizon). -In addition to these two timing filters, beliefs can be filtered by their source (see :ref:`sources`). The two timing fields follow the ISO 8601 standard and are interpreted as follows: -- "horizon": recorded at least before the fact (indicated by a positive horizon), or at most after the fact (indicated by a negative horizon). -- "prior": recorded prior to . +- ``prior``: recorded prior to . +- ``horizon``: recorded at least before the fact (indicated by a positive horizon), or at most after the fact (indicated by a negative horizon). -For example: +For example (note that you can use both fields together): .. code-block:: json @@ -387,18 +398,21 @@ For example: These fields denote that the data should have been recorded at least 6 hours before the fact (i.e. forecasts) and prior to 5 PM on August 1st 2020 (UTC). +.. note:: In addition to these two timing filters, beliefs can be filtered by their source (see :ref:`sources`). + + .. _prognoses: -Knowledge time of Prognoses -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Setting the recording time +"""""""""""""""""""""""" -Some POST endpoints have two optional fields to allow setting the time at which beliefs are recorded explicitly. +Some POST endpoints have two optional fields to allow setting the time at which beliefs are recorded in an explicit manner. This is useful to keep an accurate history of what was known at what time, especially for prognoses. -If not used, FlexMeasures will infer the prior from the arrival time of the message. +If not used, FlexMeasures will infer the belief time from the arrival time of the message. -The "prior" field (a timestamp) can be used to set a single time at which the entire prognosis was recorded. -Alternatively, the "horizon" field (a duration) can be used to set the recording times relative to each prognosed event. -In case both fields are set, the earliest possible recording time is determined and recorded for each prognosed event. +The "prior" field (a timestamp) can be used to set a single time at which the entire time series (e.g. a prognosed series) was recorded. +Alternatively, the "horizon" field (a duration) can be used to set the recording times relative to each (prognosed) event. +In case both fields are set, the earliest possible recording time is determined and recorded for each (prognosed) event. The two timing fields follow the ISO 8601 standard and are interpreted as follows: @@ -433,7 +447,7 @@ This message implies that the entire prognosis was recorded at 7:45 AM UTC, i.e. This message implies that all prognosed values were recorded 6 hours in advance. That is, the value for 1:00-1:15 PM was made at 7:15 AM, the value for 1:15-1:30 PM was made at 7:30 AM, and the value for 1:30-1:45 PM was made at 7:45 AM. -Negative horizons may also be stated (breaking with the ISO 8601 standard) to indicate a prognosis about something that has already happened (i.e. after the fact, or simply *ex post*). +Negative horizons may also be stated (breaking with the ISO 8601 standard) to indicate a belief about something that has already happened (i.e. after the fact, or simply *ex post*). For example, the following message implies that all prognosed values were made 10 minutes after the fact: .. code-block:: json @@ -449,7 +463,7 @@ For example, the following message implies that all prognosed values were made 1 "horizon": "-PT10M" } -Note that, for a horizon indicating a prognosis 10 minutes after the *start* of each 15-minute interval, the "horizon" would have been "PT5M". +Note that, for a horizon indicating a belief 10 minutes after the *start* of each 15-minute interval, the "horizon" would have been "PT5M". This denotes that the prognosed interval has 5 minutes left to be concluded. .. _resolutions: @@ -475,8 +489,8 @@ Valid units for timeseries data in version 1 of the API are "MW" only. .. _signs: -Signs of values -^^^^^^^^^^^^^^^ +Signs of power values +^^^^^^^^^^^^^^^^^^^^^ USEF recommends to use positive power values to indicate consumption and negative values to indicate production, i.e. to take the perspective of the Prosumer. diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index c6cff021e..12dd59075 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -8,7 +8,7 @@ Quickstart This section walks you through getting FlexMeasures to run with the least effort. We'll cover making a secret key, connecting a database and creating one user & one asset. -.. note:: Are you not hosting FlexMeasures, but wanting to learn how to use it? Head over to our tutorials, starting with :ref:`tut_posting_data`. +.. note:: Are you not hosting FlexMeasures, but want to learn how to use it? Head over to our tutorials, starting with :ref:`tut_posting_data`. Install FlexMeasures diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 729a72e30..937954e36 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -107,8 +107,7 @@ Price data (both observations and forecasts) can be posted to `POST /api/v2_0/p This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction. -The horizon indicates that the prices were published at 3pm on December 31st 2014 -(i.e. 33 hours ahead of midnight the next day which is the clearing time of KPX ― see below for a deeper explanation). +The ``prior`` indicates that the prices were published at 3pm on December 31st 2014 (i.e. the clearing time of the KPX day-ahead market, which is at 3 PM on the previous day ― see below for a deeper explanation). .. code-block:: json @@ -141,9 +140,9 @@ The horizon indicates that the prices were published at 3pm on December 31st 201 66.98, 58.61 ], - "start": "2015-01-01T15:00:00+09:00", + "start": "2015-01-01T00:00:00+09:00", "duration": "PT24H", - "horizon": "PT33H", + "prior": "2014-12-03T15:00:00+09:00", "unit": "KRW/kWh" }