From 5a3e592b07efa2ac66ccead76cc359101d7872ce Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Tue, 22 Mar 2022 11:25:17 +0100 Subject: [PATCH] Issue 389 publish sensor data api documentation (#390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Start API v3. Implement v3_0 endpoints /sensors [GET], /sensors/data [GET] and refactor an existing dev endpoint to become /sensors/data [POST], using modern authorization and marshmallow schemas. Also update tutorials for getting started and for posting sensor data, as well as the API introduction for v3_0 of the API. * Move sensor data API from dev to v2 and use flask-classful Signed-off-by: F.N. Claessen * Publish dev API Signed-off-by: F.N. Claessen * Add quickrefs Signed-off-by: F.N. Claessen * Add type annotations Signed-off-by: F.N. Claessen * Refactor use of auth_required in SensorAPI Signed-off-by: F.N. Claessen * Add type annotations Signed-off-by: F.N. Claessen * Add more quickrefs Signed-off-by: F.N. Claessen * Indentation Signed-off-by: F.N. Claessen * valid JSON, rather than a python dict Signed-off-by: F.N. Claessen * Move import to its preferred position Signed-off-by: F.N. Claessen * Rename and add type annotation Signed-off-by: F.N. Claessen * Allow AssetIdField and SensorIdField to be used for API requests, too, besides for CLI scripts Signed-off-by: F.N. Claessen * Update tutorial for posting data Signed-off-by: F.N. Claessen * Implement [GET] sensorData Signed-off-by: F.N. Claessen * Add example GET request Signed-off-by: F.N. Claessen * Update docstring for unit conversion Signed-off-by: F.N. Claessen * Respect the passed horizon and prior fields Signed-off-by: F.N. Claessen * Convert NaN to null (otherwise invalid JSON) Signed-off-by: F.N. Claessen * Pass event resolution to enable additional unit conversions Signed-off-by: F.N. Claessen * Smarter unit conversion for BeliefsSeries Signed-off-by: F.N. Claessen * Revert to auth_token_required instead of login_required Signed-off-by: F.N. Claessen * Deserialize to BeliefsDataFrame using post_load decorator, and run through post_load decorators in a specific sequence Signed-off-by: F.N. Claessen * Move serialization logic to schema Signed-off-by: F.N. Claessen * Fix tutorial for posting data Signed-off-by: F.N. Claessen * Introduce util function to check for a valid price unit Signed-off-by: F.N. Claessen * Comment out tutorial sections on posting data for multiple connections at once Signed-off-by: F.N. Claessen * Update tutorial for posting data: change connection to sensor and stop mentioning separate endpoints for postMeterData and postPrognosis Signed-off-by: F.N. Claessen * Add validation based on POST message type Signed-off-by: F.N. Claessen * Publish documentation for SensorDataAPI Signed-off-by: F.N. Claessen * Add validation based on GET message type Signed-off-by: F.N. Claessen * mypy Signed-off-by: F.N. Claessen * Update getting-started.rst Signed-off-by: F.N. Claessen * Undoc old endpoints Signed-off-by: F.N. Claessen * Rename with_appcontext_if_needed Signed-off-by: F.N. Claessen * refactor auth logic - separate checking access from the permission_required_for_context decorator Signed-off-by: Nicolas Höning * add modern auth for sensor data get & post, remove checking account role auth Signed-off-by: Nicolas Höning * more freedom to create inside accounts: allow all account members to create on generic assets and sensors Signed-off-by: Nicolas Höning * use a user from the same account as the sensor for sensor data tests Signed-off-by: Nicolas Höning * adapt asset tests w.r.t. to a change in dev api conftest Signed-off-by: Nicolas Höning * Remove unneeded import Signed-off-by: F.N. Claessen * GET endpoints work for logged-in users when tried in the browser with auth_required but not with auth_token_required Signed-off-by: F.N. Claessen * Add docstring Signed-off-by: F.N. Claessen * Make type a required field Signed-off-by: F.N. Claessen * Move @route to top Signed-off-by: F.N. Claessen * SensorDataDescriptionSchema doesn't need the `type` field Signed-off-by: F.N. Claessen * Revert "Undoc old endpoints" This reverts commit 852bee847d9bc7c160dd9cbdc9ca3c2e1e9804d2. * Move sensor data API documentation to v3 Signed-off-by: F.N. Claessen * Move sensor data API to v3 Signed-off-by: F.N. Claessen * Move user API to class for v3 Signed-off-by: F.N. Claessen * Move user UI to v3 Signed-off-by: F.N. Claessen * Update user API tests Signed-off-by: F.N. Claessen * Move user API tests to v3 module Signed-off-by: F.N. Claessen * Separate view for user index Signed-off-by: F.N. Claessen * Fix test Signed-off-by: F.N. Claessen * Update documentation tutorials to v3 Signed-off-by: F.N. Claessen * No trailing slash Signed-off-by: F.N. Claessen * Make `type` field in v3 Signed-off-by: F.N. Claessen * Add v3 to API index Signed-off-by: F.N. Claessen * Update introduction Signed-off-by: F.N. Claessen * Fix comment about where to request auth token Signed-off-by: F.N. Claessen * flake8 Signed-off-by: F.N. Claessen * black Signed-off-by: F.N. Claessen * Revert some documentation changes that are ahead of code changes Signed-off-by: F.N. Claessen * Use one common base route Signed-off-by: F.N. Claessen * Prefer plural base route Signed-off-by: F.N. Claessen * different shade of black Signed-off-by: F.N. Claessen * typos Signed-off-by: F.N. Claessen * Transition SensorDataAPI:post to SensorAPI:post_data and SensorDataAPI:get to SensorAPI:get_data Signed-off-by: F.N. Claessen * Rename module Signed-off-by: F.N. Claessen * Update documentation for moving from SensorDataAPI:post to SensorAPI:post_data Signed-off-by: F.N. Claessen * Update introduction Signed-off-by: F.N. Claessen * API changelog entry Signed-off-by: F.N. Claessen * Implement /sensors [GET] to replace getConnection Signed-off-by: F.N. Claessen * Complete docstring Signed-off-by: F.N. Claessen * Add extra tips to API changelog on how to upgrade from v2 to v3 Signed-off-by: F.N. Claessen * Fix check for belief timing against message type Signed-off-by: F.N. Claessen * No need to use implementations folder anymore Signed-off-by: F.N. Claessen * Contain endpoint logic within SensorAPI class Signed-off-by: F.N. Claessen * Contain endpoint logic within UserAPI class Signed-off-by: F.N. Claessen * Fix mock email and type annotations Signed-off-by: F.N. Claessen * Initialize schemas within SensorAPI and UserAPI class Signed-off-by: F.N. Claessen * Typos Signed-off-by: F.N. Claessen * Fix 404 error message Signed-off-by: F.N. Claessen * Remove unused variable declaration Signed-off-by: F.N. Claessen * Clarify use of headers in test Signed-off-by: F.N. Claessen * Concerning patching user ids and user account ids, set dump_only for these fields instead of ignoring them Signed-off-by: F.N. Claessen * Add test cases for unexpected fields Signed-off-by: F.N. Claessen * Also move patch endpoint logic into UserAPI class; this one was a bit more tricky, because the 'id' fields was being passed to the function twice somehow; use_kwargs instead of use_args was needed to avoid this Signed-off-by: F.N. Claessen * Move test utils from dev to v3_0 Signed-off-by: F.N. Claessen * Finish moving sensor data tests from dev to v3_0 Signed-off-by: F.N. Claessen * Fix asset tests in dev API by moving back some conftest logic Signed-off-by: F.N. Claessen * Instantiate schemas outside of endpoint logic to minimize response time Signed-off-by: F.N. Claessen * Add mentions of http status to docstrings Signed-off-by: F.N. Claessen * Undocument getService from the API introduction Signed-off-by: F.N. Claessen * Undocument USEF roles in introduction Signed-off-by: F.N. Claessen * Use "sensor" instead of "connection" in introduction Signed-off-by: F.N. Claessen * Undocument group notation in introduction Signed-off-by: F.N. Claessen * Update API changelog for rewrite of introduction Signed-off-by: F.N. Claessen * Update API changelog entry date Signed-off-by: F.N. Claessen Co-authored-by: Nicolas Höning --- documentation/api/change_log.rst | 33 ++- documentation/api/dev.rst | 22 ++ documentation/api/introduction.rst | 161 +++--------- documentation/api/v3_0.rst | 20 ++ documentation/getting-started.rst | 2 +- documentation/index.rst | 2 + documentation/tut/building_uis.rst | 2 +- documentation/tut/forecasting_scheduling.rst | 2 +- documentation/tut/posting_data.rst | 211 ++++++++-------- .../tut/toy-example-from-scratch.rst | 2 +- flexmeasures/api/__init__.py | 6 +- .../api/common/schemas/sensor_data.py | 196 +++++++++++++-- .../schemas/tests/test_sensor_data_schema.py | 4 +- flexmeasures/api/common/schemas/users.py | 6 +- flexmeasures/api/dev/__init__.py | 34 --- flexmeasures/api/dev/assets.py | 35 ++- flexmeasures/api/dev/sensor_data.py | 28 --- flexmeasures/api/dev/sensors.py | 20 +- flexmeasures/api/dev/tests/__init__.py | 0 flexmeasures/api/dev/tests/conftest.py | 31 +-- flexmeasures/api/dev/tests/test_assets_api.py | 2 +- .../api/dev/tests/test_assets_api_fresh_db.py | 4 +- flexmeasures/api/dev/tests/utils.py | 25 -- flexmeasures/api/v2_0/__init__.py | 4 +- flexmeasures/api/v2_0/tests/conftest.py | 19 -- flexmeasures/api/v3_0/__init__.py | 13 + flexmeasures/api/v3_0/sensors.py | 156 ++++++++++++ flexmeasures/api/v3_0/tests/__init__.py | 0 flexmeasures/api/v3_0/tests/conftest.py | 69 ++++++ .../tests/test_api_v3_0_users.py} | 43 ++-- .../tests/test_api_v3_0_users_fresh_db.py} | 2 +- .../{dev => v3_0}/tests/test_sensor_data.py | 21 +- .../tests/test_sensor_data_fresh_db.py | 6 +- flexmeasures/api/v3_0/tests/utils.py | 23 ++ flexmeasures/api/v3_0/users.py | 233 ++++++++++++++++++ flexmeasures/auth/decorators.py | 39 +-- flexmeasures/auth/policy.py | 41 +++ flexmeasures/data/models/generic_assets.py | 6 +- flexmeasures/data/models/time_series.py | 9 +- flexmeasures/data/schemas/generic_assets.py | 9 +- flexmeasures/data/schemas/sensors.py | 16 +- flexmeasures/data/schemas/users.py | 4 +- flexmeasures/data/schemas/utils.py | 13 + flexmeasures/data/services/sensors.py | 26 ++ flexmeasures/ui/crud/assets.py | 1 - flexmeasures/ui/crud/users.py | 10 +- flexmeasures/ui/tests/test_user_crud.py | 8 +- flexmeasures/ui/tests/utils.py | 7 +- flexmeasures/utils/unit_utils.py | 49 +++- 49 files changed, 1149 insertions(+), 526 deletions(-) create mode 100644 documentation/api/dev.rst create mode 100644 documentation/api/v3_0.rst delete mode 100644 flexmeasures/api/dev/sensor_data.py create mode 100644 flexmeasures/api/dev/tests/__init__.py create mode 100644 flexmeasures/api/v3_0/__init__.py create mode 100644 flexmeasures/api/v3_0/sensors.py create mode 100644 flexmeasures/api/v3_0/tests/__init__.py create mode 100644 flexmeasures/api/v3_0/tests/conftest.py rename flexmeasures/api/{v2_0/tests/test_api_v2_0_users.py => v3_0/tests/test_api_v3_0_users.py} (81%) rename flexmeasures/api/{v2_0/tests/test_api_v2_0_users_fresh_db.py => v3_0/tests/test_api_v3_0_users_fresh_db.py} (95%) rename flexmeasures/api/{dev => v3_0}/tests/test_sensor_data.py (81%) rename flexmeasures/api/{dev => v3_0}/tests/test_sensor_data_fresh_db.py (88%) create mode 100644 flexmeasures/api/v3_0/tests/utils.py create mode 100644 flexmeasures/api/v3_0/users.py create mode 100644 flexmeasures/data/services/sensors.py diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 7601753a8..a35812ddb 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,31 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-0 | 2022-03-22 +""""""""""""""""""" + +- Added REST endpoint for listing sensors: `/sensors` (GET). +- Added REST endpoint for managing sensor data: `/sensors/data` (GET, POST). +- [**Breaking change**] Switched to plural resource names for REST endpoints: `/users/` (GET, PATCH) and `/users//password-reset` (PATCH). +- [**Breaking change**] Deprecated the following endpoints: + + - *getConnection* -> use `/sensors` (GET) instead + - *getMeterData* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" + - *getPrognosis* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" + - *getService* + - *postMeterData* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" + - *postPriceData* -> use `/sensors/data` (POST) instead, replacing the "market" field with "sensor" + - *postPrognosis* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" + - *postWeatherData* -> use `/sensors/data` (POST) instead + - *restoreData* + +- Changed the Introduction section: + + - Rewrote the section on service listing for API versions to refer to the public documentation. + - Rewrote the section on entity addresses to refer to *sensors* instead of *connections*. + - Rewrote the sections on roles and sources into a combined section that refers to account roles rather than USEF roles. + - Deprecated the section on group notation. + v2.0-4 | 2022-01-04 """"""""""""""""""" @@ -42,12 +67,12 @@ v2.0-2 | 2021-04-02 v2.0-1 | 2021-02-19 """"""""""""""""""" -- REST endpoints for managing users: `/users/` (GET), `/user/` (GET, PATCH) and `/user//password-reset` (PATCH). +- Added REST endpoints for managing users: `/users/` (GET), `/user/` (GET, PATCH) and `/user//password-reset` (PATCH). v2.0-0 | 2020-11-14 """"""""""""""""""" -- REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). +- Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). v1.3-11 | 2022-01-05 @@ -80,7 +105,7 @@ v1.3-8 | 2020-04-02 *Affects all versions since v1.0*. -- [**Breaking change**, partially reverted in v1.3-9] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API version below v2.0. +- [**Breaking change**, partially reverted in v1.3-9] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API versions below v2.0. v1.3-7 | 2020-12-16 """"""""""""""""""" @@ -107,7 +132,7 @@ v1.3-5 | 2020-10-29 - Endpoints to POST meter data will now check incoming data to see if the required asset's resolution is being used ― upsampling is done if possible. These endpoints can now return the REQUIRED_INFO_MISSING status 400 response. - Endpoints to GET meter data will return data in the asset's resolution ― downsampling to the "resolution" field is done if possible. -- As they need to determine the asset, all of the mentioned POST and GET endpoints can now return the UNRECOGNIZED_ASSET status 4000 response. +- As they need to determine the asset, all of the mentioned POST and GET endpoints can now return the UNRECOGNIZED_ASSET status 400 response. v1.3-4 | 2020-06-18 """"""""""""""""""" diff --git a/documentation/api/dev.rst b/documentation/api/dev.rst new file mode 100644 index 000000000..19cd61ef9 --- /dev/null +++ b/documentation/api/dev.rst @@ -0,0 +1,22 @@ +.. _dev: + +Developer API +============= + +These endpoints are still under development and are subject to change in new releases. + +Summary +------- + +.. qrefflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.dev.assets, flexmeasures.api.dev.sensors + :order: path + :include-empty-docstring: + +API Details +----------- + +.. autoflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.dev.assets, flexmeasures.api.dev.sensors + :order: path + :include-empty-docstring: diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index be573f97a..43a9d9855 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -23,13 +23,13 @@ So if you are running FlexMeasures on your computer, it would be: https://localhost:5000/api -At Seita, we run servers for our clients at: +Let's assume we are running a server for a client at: .. code-block:: html https://company.flexmeasures.io/api -where `company` is a hosting customer of ours. All their accounts' data lives on that server. +where `company` is a client of ours. All their accounts' data lives on that server. We assume in this document that the FlexMeasures instance you want to connect to is hosted at https://company.flexmeasures.io. @@ -40,50 +40,20 @@ Let's see what the ``/api`` endpoint returns: >>> import requests >>> res = requests.get("https://company.flexmeasures.io/api") >>> res.json() - {'flexmeasures_version': '0.4.0', - 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: /api/requestAuthToken', + {'flexmeasures_version': '0.9.0', + 'message': 'For these API versions endpoints are available. An authentication token can be requested at: /api/requestAuthToken. For a list of services, see https://flexmeasures.readthedocs.io', 'status': 200, - 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0'] + 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0', 'v3_0'] } -So this tells us which API versions exist. For instance, we know that the latest API version is available at +So this tells us which API versions exist. For instance, we know that the latest API version is available at: .. code-block:: html - https://company.flexmeasures.io/api/v2_0 + https://company.flexmeasures.io/api/v3_0 -Also, we can see that a list of endpoints which are available at (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. - -**Example request** - -Let's ask which endpoints are available for meter data companies (MDC): - -.. code-block:: html - - https://company.flexmeasures.io/api/v2_0/getService?access=MDC - - -**Example response** - -.. code-block:: json - - { - "type": "GetServiceResponse", - "version": "1.0", - "services": [ - { - "name": "getMeterData", - "access": ["Aggregator", "Supplier", "MDC", "DSO", "Prosumer", "ESCo"], - "description": "Request meter reading" - }, - { - "name": "postMeterData", - "access": ["MDC"], - "description": "Send meter reading" - } - ] - } +Also, we can see that a list of endpoints is available on https://flexmeasures.readthedocs.io for each of these versions. .. _api_auth: @@ -131,28 +101,13 @@ which gives a response like this if the credentials are correct: .. note:: Each access token has a limited lifetime, see :ref:`auth`. -Roles ------ - -We distinguish the following roles with different access rights to the individual services. Capitalised roles are defined by USEF: - -- public -- user -- admin -- Aggregator -- Supplier: an energy retailer (see :ref:`supplier`) -- Prosumer: owner of a grid connection (see :ref:`prosumer`) -- ESCo: an energy service company (see :ref:`esco`) -- MDC: a meter data company (see :ref:`mdc`) -- DSO: a distribution system operator (see :ref:`dso`) - .. _sources: Sources ------- Requests for data may limit the data selection by specifying a source, for example, a specific user. -USEF roles are also valid source selectors. +Account roles are also valid source selectors. For example, to obtain data originating from either a meter data company or user 42, include the following: .. code-block:: json @@ -161,6 +116,8 @@ For example, to obtain data originating from either a meter data company or user "sources": ["MDC", "42"], } +Here, "MDC" is the name of the account role for meter data companies. + Notation -------- All requests and responses to and from the web service should be valid JSON messages. @@ -179,27 +136,26 @@ Throughout this document, keys are written in singular if a single value is list The API, however, does not distinguish between singular and plural key notation. -Connections and entity addresses +Sensors and entity addresses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A connection represents an end point of the grid, at which an electricity sensor (power meter) is located. -Connections should be identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], -which is mostly taken from IETF RFC 3720 [2]: +All sensors are identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], +which is mostly taken from IETF RFC 3720 [2]. This is the complete structure of an EA1 address: .. code-block:: json { - "connection": "ea1.{date code}.{reversed domain name}:{locally unique string}" + "sensor": "ea1.{date code}.{reversed domain name}:{locally unique string}" } -Here is a full example for a FlexMeasures connection address: +Here is a full example for an entity address of a sensor in FlexMeasures: .. code-block:: json { - "connection": "ea1.2021-02.io.flexmeasures.company:fm1.73" + "sensor": "ea1.2021-02.io.flexmeasures.company:fm1.73" } where FlexMeasures runs at `company.flexmeasures.io` (which the current domain owner started using in February 2021), and the locally unique string uses the `fm1` scheme (see below) to identify sensor ID 73. @@ -254,7 +210,7 @@ It uses the fact that all FlexMeasures sensors have unique IDs. .. todo:: UDI events are not yet modelled in the fm1 scheme The ``fm0`` scheme is the original scheme. -It identified different types of sensors (such as connections, weather sensors and markets) in different ways. +It identified different types of sensors (such as grid connections, weather sensors and markets) in different ways. The ``fm0`` scheme has been deprecated for the most part and is no longer supported officially. Only UDI events still need to be sent using the fm0 scheme. @@ -264,69 +220,6 @@ Only UDI events still need to be sent using the fm0 scheme. ea1.2021-01.io.flexmeasures:fm0.::: -Groups -^^^^^^ - -Data such as measurements, load prognoses and tariffs are usually stated per group of connections. -When the attributes "start", "duration" and "unit" are stated outside of "groups" they are inherited by each of the individual groups. For example: - -.. code-block:: json - - { - "groups": [ - { - "connections": [ - "ea1.2021-02.io.flexmeasures.company:fm1.71", - "ea1.2021-02.io.flexmeasures.company:fm1.72" - ], - "values": [ - 306.66, - 306.66, - 0, - 0, - 306.66, - 306.66 - ] - }, - { - "connection": "ea1.2021-02.io.flexmeasures.company:fm1.73" - "values": [ - 306.66, - 0, - 0, - 0, - 306.66, - 306.66 - ] - } - ], - "start": "2016-05-01T12:45:00Z", - "duration": "PT1H30M", - "unit": "MW" - } - -In case of a single group of connections, the message may be flattened to: - -.. code-block:: json - - { - "connections": [ - "ea1.2021-02.io.flexmeasures.company:fm1.71", - "ea1.2021-02.io.flexmeasures.company:fm1.72" - ], - "values": [ - 306.66, - 306.66, - 0, - 0, - 306.66, - 306.66 - ], - "start": "2016-05-01T12:45:00Z", - "duration": "PT1H30M", - "unit": "MW" - } - Timeseries ^^^^^^^^^^ @@ -374,10 +267,10 @@ Technically, this is equal to: This intuitive convention allows us to reduce communication by sending univariate timeseries as arrays. -Notation for v1 -""""""""""""""" +Notation for v1, v2 and v3 +"""""""""""""""""""""""""" -For version 1 and 2 of the API, only equidistant timeseries data is expected to be communicated. Therefore: +For version 1, 2 and 3 of the API, only equidistant timeseries data is expected to be communicated. Therefore: - only the array notation should be used (first notation from above), - "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and @@ -496,19 +389,25 @@ Resolutions Specifying a resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the resolution by dividing the duration by the number of values. -When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the assets which are posted to. If these can't be matched (through upsampling), an error will occur. +When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the sensors which are posted to. If these can't be matched (through upsampling), an error will occur. GET requests (such as *getMeterData*) return data in the resolution which the sensor is configured for. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. The specified resolution needs to be a multiple -of the asset's resolution, e.g. hourly or daily values if the asset's resolution is 15 minutes. +of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. .. _units: Units ^^^^^ -Valid units for timeseries data in version 1 of the API are "MW" only. +From API version 3 onwards, we are much more flexible with sent units. +A valid unit for timeseries data is any unit that is convertible to the configured sensor unit registered in FlexMeasures. +So, for example, you can send timeseries data with "W" unit to a "kW" sensor. +And if you wish to do so, you can even send a timeseries with "kWh" unit to a "kW" sensor. +In this case, FlexMeasures will convert the data using the resolution of the timeseries. + +For API versions 1 and 2, the unit sent needs to be an exact match with the sensor unit, and only "MW" is allowed for power sensors. .. _signs: diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst new file mode 100644 index 000000000..0d589fb4a --- /dev/null +++ b/documentation/api/v3_0.rst @@ -0,0 +1,20 @@ +.. _v3_0: + +Version 3.0 +=========== + +Summary +------- + +.. qrefflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v3_0.implementations.sensors, flexmeasures.api.v3_0.implementations.users + :order: path + :include-empty-docstring: + +API Details +----------- + +.. autoflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v3_0.implementations.sensors, flexmeasures.api.v3_0.implementations.users + :order: path + :include-empty-docstring: diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 200046fe8..ef0442794 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -195,7 +195,7 @@ First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` This assumes you have a file `my-data.csv` with measurements, which was exported from some legacy database, and that the data is about our sensor with ID 1. This command has many options, so do use its ``--help`` function. -Second, you can use the `POST /api/v2_0/postMeterData `_ endpoint in the FlexMeasures API to send meter data. +Second, you can use the `POST /api/v3_0/sensors/data `_ endpoint in the FlexMeasures API to send meter data. Finally, you can tell FlexMeasures to create forecasts for your meter data with the ``flexmeasures add forecasts`` command, here is an example: diff --git a/documentation/index.rst b/documentation/index.rst index d0854e3cd..9444fa478 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -128,11 +128,13 @@ The platform operator of FlexMeasures can be an Aggregator. :maxdepth: 1 api/introduction + api/v3_0 api/v2_0 api/v1_3 api/v1_2 api/v1_1 api/v1 + api/dev api/change_log .. toctree:: diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst index 08856e594..2041b943c 100644 --- a/documentation/tut/building_uis.rst +++ b/documentation/tut/building_uis.rst @@ -20,7 +20,7 @@ This tutorial will show how the FlexMeasures API can be used from JavaScript to Get an authentication token ----------------------- -FlexMeasures provides the `POST /api/v2_0/requestAuthToken <../api/v2_0.html#post--api-v2_0-requestAuthToken>`_ endpoint, as discussed in :ref:`api_auth`. +FlexMeasures provides the `POST /api/requestAuthToken <../api/v2_0.html#post--api-v2_0-requestAuthToken>`_ endpoint, as discussed in :ref:`api_auth`. Here is a JavaScript function to call it: .. code-block:: JavaScript diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 76ec02164..7c51073d0 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -116,7 +116,7 @@ Getting power forecasts (prognoses) Prognoses (the USEF term used for power forecasts) are used by FlexMeasures to determine the best control signals to valorise on balancing opportunities. -You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. +You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. Getting them might be useful if you want to use prognoses in your own system, or to check their accuracy against meter data, i.e. the realised power measurements. The FlexMeasures UI also lists forecast accuracy, and visualises prognoses and meter data next to each other. diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index c15cd4e9c..60e21b161 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -26,24 +26,37 @@ Prerequisites .. note:: To address assets and sensors, these tutorials assume entity addresses valid in the namespace ``fm1``. See :ref:`api_introduction` for more explanations. -Posting price data ------------------- +.. _posting_sensor_data: + +Posting sensor data +------------------- + +Sensor data (both observations and forecasts) can be posted to `POST /api/v3_0/sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_. +This endpoint represents the basic method of getting time series data into FlexMeasures via API. +It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that have happened in the past or will happen in the future. +Some examples: -Price data (both observations and forecasts) can be posted to `POST /api/v2_0/postPriceData <../api/v2_0.html#post--api-v2_0-postPriceData>`_. The URL might look like this: +- readings from electricity and gas meters +- readings from temperature and pressure sensors +- state of charge of a battery +- estimated availability of parking spots +- price forecasts + +The exact URL will depend on your domain name, and will look approximately like this: .. code-block:: html - https://company.flexmeasures.io/api//postPriceData + [POST] https://company.flexmeasures.io/api//sensors/data -This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day +This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. The ``prior`` indicates that the prices were published at 3pm on December 31st 2014 (i.e. the clearing time of the KPX day-ahead market, which is at 3 PM on the previous day ― see below for a deeper explanation). .. code-block:: json { - "type": "PostPriceDataRequest", - "market": "ea1.2021-01.io.flexmeasures.company:fm1.16", + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.16", "values": [ 52.37, 51.14, @@ -78,41 +91,40 @@ The ``prior`` indicates that the prices were published at 3pm on December 31st 2 Note how the resolution of the data comes out at 60 minutes when you divide the duration by the number of data points. If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match or, if that is not possible, complain. +Likewise, if the data unit does not match the sensor’s unit, FlexMeasures will attempt to convert the data or, if that is not possible, complain. Posting power data ------------------ For power data, USEF specifies separate message types for observations and forecasts. -Correspondingly, FlexMeasures uses separate endpoints to communicate these messages. -Observations of power data can be posted to `POST /api/v2_0/postMeterData <../api/v2_0.html#post--api-v2_0-postMeterData>`_. The URL might look like this: - -.. code-block:: html +Correspondingly, we allow the following message types to be used with the [POST] /sensors/data endpoint (see :ref:`posting_sensor_data`): - https://company.flexmeasures.io/api//postMeterData - -while forecasts of power data can be posted to `POST /api/v2_0/postPrognosis <../api/v2_0.html#post--api-v2_0-postPrognosis>`_. The URL might look like this: - -.. code-block:: html +.. code-block:: json - https://company.flexmeasures.io/api//postPrognosis + { + "type": "PostMeterDataRequest" + } -For both endpoints, power data can be posted in various ways. -The following examples assume that the endpoint for power data observations (i.e. meter data) is used. +.. code-block:: json -.. todo:: For the time being, only one rate unit (MW) can be used to post power values. + { + "type": "PostPrognosisRequest" + } +For these message types, FlexMeasures validates whether the data unit is suitable for communicating power data. +Additionally, we validate whether meter data lies in the past, and prognoses lie in the future. -Single value, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Single value, single sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A single average power value for a 15-minute time interval for a single connection, posted 5 minutes after realisation. +A single average power value for a 15-minute time interval for a single sensor, posted 5 minutes after realisation. .. code-block:: json { - "type": "PostMeterDataRequest", - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", "value": 220, "start": "2015-01-01T00:00:00+00:00", "duration": "PT0H15M", @@ -120,16 +132,16 @@ A single average power value for a 15-minute time interval for a single connecti "unit": "MW" } -Multiple values, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Multiple values, single sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after each realisation. +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single sensor, posted 5 minutes after each realisation. .. code-block:: json { - "type": "PostMeterDataRequest", - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", "values": [ 220, 210, @@ -141,85 +153,88 @@ Multiple values (indicating a univariate timeseries) for 15-minute time interval "unit": "MW" } -Single identical value, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + todo: uncomment whenever the new sensor data API supports sending data for multiple sensors in one message -Single identical value for a 15-minute time interval for two connections, posted 5 minutes after realisation. -Please note that both connections consumed at 10 MW, i.e. the value does not represent the total of the two connections. -We recommend to use this notation for zero values only. + Single identical value, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Single identical value for a 15-minute time interval for two sensors, posted 5 minutes after realisation. + Please note that both sensors consumed at 10 MW, i.e. the value does not represent the total of the two sensors. + We recommend to use this notation for zero values only. - { - "type": "PostMeterDataRequest", - "connections": [ - "ea1.2021-01.io.flexmeasures.company:fm1.1", - "ea1.2021-01.io.flexmeasures.company:fm1.2" - ], - "value": 10, - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json -Single different values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + { + "type": "PostSensorDataRequest", + "sensors": [ + "ea1.2021-01.io.flexmeasures.company:fm1.1", + "ea1.2021-01.io.flexmeasures.company:fm1.2" + ], + "value": 10, + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } -Single different values for a 15-minute time interval for two connections, posted 5 minutes after realisation. + Single different values, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Single different values for a 15-minute time interval for two sensors, posted 5 minutes after realisation. - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", - "value": 220 - }, - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.2", - "value": 300 - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json -Multiple values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + { + "type": "PostSensorDataRequest", + "groups": [ + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "value": 220 + }, + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.2", + "value": 300 + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after each realisation. + Multiple values, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two sensors, posted 5 minutes after each realisation. - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", - "values": [ - 220, - 210, - 200 - ] - }, - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.2", - "values": [ - 300, - 303, - 306 - ] - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H45M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json + + { + "type": "PostSensorDataRequest", + "groups": [ + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "values": [ + 220, + 210, + 200 + ] + }, + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.2", + "values": [ + 300, + 303, + 306 + ] + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H45M", + "horizon": "-PT5M", + "unit": "MW" + } .. _observations_vs_forecasts diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index a1d17a2fb..bba832372 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -108,7 +108,7 @@ If you want, you can inspect what you created: Yes, that is quite a large battery :) -.. note:: Obviously, you can use the ``flexmeasures`` command to create your own, custom account and assets. See :ref:`cli`. And to create, edit or read asset data via the API, see :ref:`v2_0`. +.. note:: Obviously, you can use the ``flexmeasures`` command to create your own, custom account and assets. See :ref:`cli`. And to create, edit or read asset data via the API, see :ref:`v3_0`. We can also look at the battery asset in the UI of FlexMeasures (start FlexMeasures with ``flexmeasures run``, username is "toy-user@flexmeasures.io", password is "toy-password"): diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 3b104731b..dd7164f4e 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -73,9 +73,9 @@ def get_versions() -> dict: """ response = { "message": "For these API versions a public endpoint is available, listing its service. For example: " - "/api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: " + "/api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: " "/api/requestAuthToken", - "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0"], + "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0", "v3_0"], "flexmeasures_version": flexmeasures_version, } return response @@ -110,6 +110,7 @@ def register_at(app: Flask): from flexmeasures.api.v1_2 import register_at as v1_2_register_at from flexmeasures.api.v1_3 import register_at as v1_3_register_at from flexmeasures.api.v2_0 import register_at as v2_0_register_at + from flexmeasures.api.v3_0 import register_at as v3_0_register_at from flexmeasures.api.dev import register_at as dev_register_at v1_register_at(app) @@ -117,4 +118,5 @@ def register_at(app: Flask): v1_2_register_at(app) v1_3_register_at(app) v2_0_register_at(app) + v3_0_register_at(app) dev_register_at(app) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 2b6264f51..38f8a002e 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -2,22 +2,27 @@ from typing import List, Union from flask_login import current_user +from isodate import datetime_isoformat, duration_isoformat from marshmallow import fields, post_load, validates_schema, ValidationError -from marshmallow.validate import Equal, OneOf +from marshmallow.validate import OneOf from marshmallow_polyfield import PolyField from timely_beliefs import BeliefsDataFrame import pandas as pd from flexmeasures.data import ma from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.time_series import Sensor from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField +from flexmeasures.data.services.time_series import simplify_index from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.unit_utils import ( convert_units, units_are_convertible, + is_energy_price_unit, ) +from flexmeasures.auth.policy import check_access class SingleValueField(fields.Float): @@ -60,13 +65,9 @@ def select_schema_to_ensure_list_of_floats( class SensorDataDescriptionSchema(ma.Schema): """ - Describing sensor data (i.e. in a GET request). - - TODO: when we want to support other entity types with this - schema (assets/weather/markets or actuators), we'll need some re-design. + Schema describing sensor data (specifically, the sensor and the timing of the data). """ - type = fields.Str(required=True, validate=Equal("GetSensorDataRequest")) sensor = SensorField(required=True, entity_type="sensor", fm_scheme="fm1") start = AwareDateTimeField(required=True, format="iso") duration = DurationField(required=True) @@ -74,15 +75,6 @@ class SensorDataDescriptionSchema(ma.Schema): prior = AwareDateTimeField(required=False, format="iso") unit = fields.Str(required=True) - @validates_schema - def check_user_rights_against_sensor(self, data, **kwargs): - """If the user is a Prosumer and the sensor belongs to an asset - over which the Prosumer has no ownership, raise a ValidationError. - """ - # todo: implement check once sensors can belong to an asset - # https://github.com/FlexMeasures/flexmeasures/issues/155 - pass - @validates_schema def check_schema_unit_against_sensor_unit(self, data, **kwargs): """Allows units compatible with that of the sensor. @@ -103,7 +95,116 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): ) -class SensorDataSchema(SensorDataDescriptionSchema): +class GetSensorDataSchema(SensorDataDescriptionSchema): + + # Optional field that can be used for extra validation + type = fields.Str( + required=False, + validate=OneOf( + [ + "GetSensorDataRequest", + "GetMeterDataRequest", + "GetPrognosisRequest", + "GetPriceDataRequest", + ] + ), + ) + + @validates_schema + def check_user_may_read(self, data, **kwargs): + check_access(data["sensor"], "read") + + @validates_schema + def check_schema_unit_against_type(self, data, **kwargs): + requested_unit = data["unit"] + _type = data.get("type", None) + if ( + _type + in ( + "GetMeterDataRequest", + "GetPrognosisRequest", + ) + and not units_are_convertible(requested_unit, "MW") + ): + raise ValidationError( + f"The unit requested for this message type should be convertible from MW, got incompatible unit: {requested_unit}" + ) + elif _type == "GetPriceDataRequest" and not is_energy_price_unit( + requested_unit + ): + raise ValidationError( + f"The unit requested for this message type should be convertible from an energy price unit, got incompatible unit: {requested_unit}" + ) + + @post_load + def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: + """Turn the de-serialized and validated data description into a response. + + Specifically, this function: + - queries data according to the given description + - converts to a single deterministic belief per event + - ensures the response respects the requested time frame + - converts values to the requested unit + """ + sensor: Sensor = sensor_data_description["sensor"] + start = sensor_data_description["start"] + duration = sensor_data_description["duration"] + end = sensor_data_description["start"] + duration + unit = sensor_data_description["unit"] + + # Post-load configuration of belief timing against message type + horizons_at_least = sensor_data_description.get("horizon", None) + horizons_at_most = None + _type = sensor_data_description.get("type", None) + if _type == "GetMeterDataRequest": + horizons_at_most = timedelta(0) + elif _type == "GetPrognosisRequest": + if horizons_at_least is None: + horizons_at_least = timedelta(0) + else: + # If the horizon field is used, ensure we still respect the minimum horizon for prognoses + horizons_at_least = max(horizons_at_least, timedelta(0)) + + df = simplify_index( + sensor.search_beliefs( + event_starts_after=start, + event_ends_before=end, + horizons_at_least=horizons_at_least, + horizons_at_most=horizons_at_most, + beliefs_before=sensor_data_description.get("prior", None), + one_deterministic_belief_per_event=True, + as_json=False, + ) + ) + + # Convert to desired time range + index = pd.date_range( + start=start, end=end, freq=sensor.event_resolution, closed="left" + ) + df = df.reindex(index) + + # Convert to desired unit + values: pd.Series = convert_units( # type: ignore + df["event_value"], + from_unit=sensor.unit, + to_unit=unit, + ) + + # Convert NaN to null + values = values.where(pd.notnull(values), None) + + # Form the response + response = dict( + values=values.tolist(), + start=datetime_isoformat(start), + duration=duration_isoformat(duration), + unit=unit, + ) + + return response + + +class PostSensorDataSchema(SensorDataDescriptionSchema): """ This schema includes data, so it can be used for POST requests or GET responses. @@ -112,8 +213,18 @@ class SensorDataSchema(SensorDataDescriptionSchema): (sets a resolution parameter which we can pass to the data collection function). """ + # Optional field that can be used for extra validation type = fields.Str( - validate=OneOf(["PostSensorDataRequest", "GetSensorDataResponse"]) + required=False, + validate=OneOf( + [ + "PostSensorDataRequest", + "PostMeterDataRequest", + "PostPrognosisRequest", + "PostPriceDataRequest", + "PostWeatherDataRequest", + ] + ), ) values = PolyField( deserialization_schema_selector=select_schema_to_ensure_list_of_floats, @@ -121,6 +232,30 @@ class SensorDataSchema(SensorDataDescriptionSchema): many=False, ) + @validates_schema + def check_user_may_create(self, data, **kwargs): + check_access(data["sensor"], "create-children") + + @validates_schema + def check_schema_unit_against_type(self, data, **kwargs): + posted_unit = data["unit"] + _type = data.get("type", None) + if ( + _type + in ( + "PostMeterDataRequest", + "PostPrognosisRequest", + ) + and not units_are_convertible(posted_unit, "MW") + ): + raise ValidationError( + f"The unit required for this message type should be convertible to MW, got incompatible unit: {posted_unit}" + ) + elif _type == "PostPriceDataRequest" and not is_energy_price_unit(posted_unit): + raise ValidationError( + f"The unit required for this message type should be convertible to an energy price unit, got incompatible unit: {posted_unit}" + ) + @validates_schema def check_resolution_compatibility_of_values(self, data, **kwargs): inferred_resolution = data["duration"] / len(data["values"]) @@ -134,7 +269,25 @@ def check_resolution_compatibility_of_values(self, data, **kwargs): ) @post_load() - def possibly_convert_units(self, data, **kwargs): + def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: + """If needed, upsample and convert units, then deserialize to a BeliefsDataFrame.""" + data = self.possibly_upsample_values(data) + data = self.possibly_convert_units(data) + bdf = self.load_bdf(data) + + # Post-load validation against message type + _type = data.get("type", None) + if _type == "PostMeterDataRequest": + if any(h > timedelta(0) for h in bdf.belief_horizons): + raise ValidationError("Meter data must lie in the past.") + elif _type == "PostPrognosisRequest": + if any(h < timedelta(0) for h in bdf.belief_horizons): + raise ValidationError("Prognoses must lie in the future.") + + return bdf + + @staticmethod + def possibly_convert_units(data): """ Convert values if needed, to fit the sensor's unit. Marshmallow runs this after validation. @@ -147,8 +300,8 @@ def possibly_convert_units(self, data, **kwargs): ) return data - @post_load() - def possibly_upsample_values(self, data, **kwargs): + @staticmethod + def possibly_upsample_values(data): """ Upsample the data if needed, to fit to the sensor's resolution. Marshmallow runs this after validation. @@ -169,7 +322,8 @@ def possibly_upsample_values(self, data, **kwargs): ) return data - def load_bdf(sensor_data) -> BeliefsDataFrame: + @staticmethod + def load_bdf(sensor_data: dict) -> BeliefsDataFrame: """ Turn the de-serialized and validated data into a BeliefsDataFrame. """ diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 2862fc558..07e859e10 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -4,7 +4,7 @@ from flexmeasures.api.common.schemas.sensor_data import ( SingleValueField, - SensorDataSchema, + PostSensorDataSchema, ) @@ -34,7 +34,7 @@ def test_value_field_deserialization( exp_deserialization_output, ): """Testing straightforward cases""" - vf = SensorDataSchema._declared_fields["values"] + vf = PostSensorDataSchema._declared_fields["values"] deser = vf.deserialize(deserialization_input) assert deser == exp_deserialization_output diff --git a/flexmeasures/api/common/schemas/users.py b/flexmeasures/api/common/schemas/users.py index 84e14c943..4470e9968 100644 --- a/flexmeasures/api/common/schemas/users.py +++ b/flexmeasures/api/common/schemas/users.py @@ -7,7 +7,7 @@ class AccountIdField(fields.Integer): """ - Field that represents an account ID. It de-serializes from the account id to an account instance. + Field that represents an account ID. It deserializes from the account id to an account instance. """ def _deserialize(self, account_id: str, attr, obj, **kwargs) -> Account: @@ -30,7 +30,7 @@ def load_current(cls): class UserIdField(fields.Integer): """ - Field that represents a user ID. It de-serializes from the user id to a user instance. + Field that represents a user ID. It deserializes from the user id to a user instance. """ def __init__(self, *args, **kwargs): @@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs): def _deserialize(self, user_id: int, attr, obj, **kwargs) -> User: user: User = User.query.filter_by(id=int(user_id)).one_or_none() if user is None: - raise abort(404, f"User {id} not found") + raise abort(404, f"User {user_id} not found") return user def _serialize(self, user: User, attr, data, **kwargs) -> int: diff --git a/flexmeasures/api/dev/__init__.py b/flexmeasures/api/dev/__init__.py index d4f90bf9e..f82da36ce 100644 --- a/flexmeasures/api/dev/__init__.py +++ b/flexmeasures/api/dev/__init__.py @@ -1,7 +1,4 @@ from flask import Flask -from flask_security import auth_token_required - -from flexmeasures.auth.decorators import account_roles_accepted def register_at(app: Flask): @@ -9,39 +6,8 @@ def register_at(app: Flask): from flexmeasures.api.dev.sensors import SensorAPI from flexmeasures.api.dev.assets import AssetAPI - from flexmeasures.api.dev.sensor_data import post_data as post_sensor_data_impl dev_api_prefix = "/api/dev" SensorAPI.register(app, route_prefix=dev_api_prefix) AssetAPI.register(app, route_prefix=dev_api_prefix) - - @app.route(f"{dev_api_prefix}/sensorData", methods=["POST"]) - @auth_token_required - @account_roles_accepted("MDC", "Prosumer") - def post_sensor_data(): - """ - Post sensor data to FlexMeasures. - - For example: - - { - "type": "PostSensorDataRequest", - "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", - "values": [-11.28, -11.28, -11.28, -11.28], - "start": "2021-06-07T00:00:00+02:00", - "duration": "PT1H", - "unit": "m³/h", - } - - The above request posts four values for a duration of one hour, where the first - event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. - - The sensor is the one with ID=1. - The unit has to match the sensor's required unit. - The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. - """ - return post_sensor_data_impl() - - # TODO: add GET /sensorData diff --git a/flexmeasures/api/dev/assets.py b/flexmeasures/api/dev/assets.py index 94245bb3d..1b2650ed7 100644 --- a/flexmeasures/api/dev/assets.py +++ b/flexmeasures/api/dev/assets.py @@ -6,7 +6,7 @@ from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data import db from flexmeasures.data.models.user import Account -from flexmeasures.data.models.generic_assets import GenericAsset as AssetModel +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField @@ -36,7 +36,10 @@ class AssetAPI(FlaskView): @permission_required_for_context("read", arg_name="account") @as_json def index(self, account: Account): - """List all assets owned by a certain account.""" + """List all assets owned by a certain account. + + .. :quickref: Asset; Download asset list + """ return assets_schema.dump(account.generic_assets), 200 @route("/", methods=["POST"]) @@ -44,9 +47,12 @@ def index(self, account: Account): "create-children", arg_loader=AccountIdField.load_current ) @use_args(AssetSchema()) - def post(self, asset_data): - """Create new asset""" - asset = AssetModel(**asset_data) + def post(self, asset_data: dict): + """Create new asset. + + .. :quickref: Asset; Create a new asset + """ + asset = GenericAsset(**asset_data) db.session.add(asset) db.session.commit() return asset_schema.dump(asset), 201 @@ -56,7 +62,10 @@ def post(self, asset_data): @permission_required_for_context("read", arg_name="asset") @as_json def fetch_one(self, id, asset): - """Fetch a given asset""" + """Fetch a given asset. + + .. :quickref: Asset; Get an asset + """ return asset_schema.dump(asset), 200 @route("/", methods=["PATCH"]) @@ -64,8 +73,11 @@ def fetch_one(self, id, asset): @use_kwargs({"db_asset": AssetIdField(data_key="id")}, location="path") @permission_required_for_context("update", arg_name="db_asset") @as_json - def patch(self, asset_data, id, db_asset): - """Update an asset given its identifier""" + def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): + """Update an asset given its identifier. + + .. :quickref: Asset; Update an asset + """ ignored_fields = ["id", "account_id"] for k, v in [(k, v) for k, v in asset_data.items() if k not in ignored_fields]: setattr(db_asset, k, v) @@ -77,8 +89,11 @@ def patch(self, asset_data, id, db_asset): @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") @permission_required_for_context("delete", arg_name="asset") @as_json - def delete(self, id, asset): - """Delete an asset given its identifier""" + def delete(self, id: int, asset: GenericAsset): + """Delete an asset given its identifier. + + .. :quickref: Asset; Delete an asset + """ asset_name = asset.name db.session.delete(asset) db.session.commit() diff --git a/flexmeasures/api/dev/sensor_data.py b/flexmeasures/api/dev/sensor_data.py deleted file mode 100644 index a8dd758f6..000000000 --- a/flexmeasures/api/dev/sensor_data.py +++ /dev/null @@ -1,28 +0,0 @@ -from webargs.flaskparser import use_args - -from flexmeasures.api.common.schemas.sensor_data import SensorDataSchema -from flexmeasures.api.common.utils.api_utils import save_and_enqueue - - -@use_args( - SensorDataSchema(), - location="json", -) -def post_data(sensor_data): - """POST to /sensorData - - Experimental dev feature which uses timely-beliefs - to create and save the data structure. - """ - beliefs = SensorDataSchema.load_bdf(sensor_data) - response, code = save_and_enqueue(beliefs) - response.update(type="PostSensorDataResponse") - return response, code - - -def get_data(): - """GET from /sensorData""" - # - use data.models.time_series.Sensor::search_beliefs() - might need to add a belief_horizon parameter - # - create the serialize method on the schema, to turn the resulting BeliefsDataFrame - # to the JSON the API should respond with. - pass diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 2d19edf3f..f17e01b2e 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -18,8 +18,8 @@ class SensorAPI(FlaskView): """ route_base = "/sensor" + decorators = [auth_required()] - @auth_required() @route("//chart/") @use_kwargs( { @@ -34,12 +34,14 @@ class SensorAPI(FlaskView): }, location="query", ) - def get_chart(self, id, **kwargs): - """GET from /sensor//chart""" + def get_chart(self, id: int, **kwargs): + """GET from /sensor//chart + + .. :quickref: Chart; Download a chart with time series + """ sensor = get_sensor_or_abort(id) return json.dumps(sensor.chart(**kwargs)) - @auth_required() @route("//chart_data/") @use_kwargs( { @@ -50,17 +52,21 @@ def get_chart(self, id, **kwargs): }, location="query", ) - def get_chart_data(self, id, **kwargs): + def get_chart_data(self, id: int, **kwargs): """GET from /sensor//chart_data + .. :quickref: Chart; Download time series for use in charts + Data for use in charts (in case you have the chart specs already). """ sensor = get_sensor_or_abort(id) return sensor.search_beliefs(as_json=True, **kwargs) - @auth_required() def get(self, id: int): - """GET from /sensor/""" + """GET from /sensor/ + + .. :quickref: Chart; Download sensor attributes for use in charts + """ sensor = get_sensor_or_abort(id) attributes = ["name", "timezone", "timerange"] return {attr: getattr(sensor, attr) for attr in attributes} diff --git a/flexmeasures/api/dev/tests/__init__.py b/flexmeasures/api/dev/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 623bf574c..6887661e2 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -1,8 +1,6 @@ -from datetime import timedelta - import pytest -from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset +from flexmeasures.api.v3_0.tests.conftest import add_gas_sensor from flexmeasures.data.models.time_series import Sensor @@ -12,7 +10,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) - add_gas_sensor(db, setup_roles_users["Test Prosumer User 2"]) + add_gas_sensor(db, setup_roles_users["Test Supplier User"]) @pytest.fixture(scope="function") @@ -25,27 +23,4 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Prosumer User 2"]) - - -def add_gas_sensor(db, test_supplier): - incineration_type = GenericAssetType( - name="waste incinerator", - ) - db.session.add(incineration_type) - db.session.flush() - incineration_asset = GenericAsset( - name="incineration line", - generic_asset_type=incineration_type, - account_id=test_supplier.account_id, - ) - db.session.add(incineration_asset) - db.session.flush() - gas_sensor = Sensor( - name="some gas sensor", - unit="m³/h", - event_resolution=timedelta(minutes=10), - generic_asset=incineration_asset, - ) - db.session.add(gas_sensor) - gas_sensor.owner = test_supplier + add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) diff --git a/flexmeasures/api/dev/tests/test_assets_api.py b/flexmeasures/api/dev/tests/test_assets_api.py index d28422569..646009bfe 100644 --- a/flexmeasures/api/dev/tests/test_assets_api.py +++ b/flexmeasures/api/dev/tests/test_assets_api.py @@ -70,7 +70,7 @@ def test_get_asset_nonaccount_access(client, setup_api_test_data): assert "not found" in asset_response.json["message"] -@pytest.mark.parametrize("account_name, num_assets", [("Prosumer", 2), ("Supplier", 1)]) +@pytest.mark.parametrize("account_name, num_assets", [("Prosumer", 1), ("Supplier", 2)]) def test_get_assets( client, setup_api_test_data, setup_accounts, account_name, num_assets ): diff --git a/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py b/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py index 119cb5ad0..676f9dd0f 100644 --- a/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py @@ -34,8 +34,8 @@ def test_post_an_asset_as_admin(client, setup_api_fresh_test_data, admin_kind): def test_edit_an_asset(client, setup_api_fresh_test_data): - with AccountContext("Test Prosumer Account") as prosumer: - existing_asset = prosumer.generic_assets[1] + with AccountContext("Test Supplier Account") as supplier: + existing_asset = supplier.generic_assets[1] post_data = dict(latitude=10, id=999) # id will be ignored auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") diff --git a/flexmeasures/api/dev/tests/utils.py b/flexmeasures/api/dev/tests/utils.py index 70ae69749..aecd33834 100644 --- a/flexmeasures/api/dev/tests/utils.py +++ b/flexmeasures/api/dev/tests/utils.py @@ -1,28 +1,3 @@ -from flexmeasures.data.models.time_series import Sensor - - -def make_sensor_data_request_for_gas_sensor( - num_values: int = 6, duration: str = "PT1H", unit: str = "m³" -) -> dict: - """Creates request to post sensor data for a gas sensor. - This particular gas sensor measures units of m³/h with a 10-minute resolution. - """ - sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() - message: dict = { - "type": "PostSensorDataRequest", - "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "values": num_values * [-11.28], - "start": "2021-06-07T00:00:00+02:00", - "duration": duration, - "horizon": "PT0H", - "unit": unit, - } - if num_values == 1: - # flatten [] to - message["values"] = message["values"][0] - return message - - def get_asset_post_data(account_id: int = 1, asset_type_id: int = 1) -> dict: post_data = { "name": "Test battery 2", diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index 06f4d23b9..b8c24c896 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -8,4 +8,6 @@ def register_at(app: Flask): import flexmeasures.api.v2_0.routes # noqa: F401 this is necessary to load the endpoints - app.register_blueprint(flexmeasures_api, url_prefix="/api/v2_0") + v2_0_api_prefix = "/api/v2_0" + + app.register_blueprint(flexmeasures_api, url_prefix=v2_0_api_prefix) diff --git a/flexmeasures/api/v2_0/tests/conftest.py b/flexmeasures/api/v2_0/tests/conftest.py index cc66035be..305a36400 100644 --- a/flexmeasures/api/v2_0/tests/conftest.py +++ b/flexmeasures/api/v2_0/tests/conftest.py @@ -1,5 +1,3 @@ -from flask_security import SQLAlchemySessionUserDatastore -from flask_security.utils import hash_password import pytest @@ -13,20 +11,3 @@ def setup_api_test_data(db, setup_roles_users, add_market_prices, add_battery_as # Add battery asset battery = add_battery_assets["Test battery"] battery.owner = setup_roles_users["Test Prosumer User 2"] - - -@pytest.fixture(scope="module") -def setup_inactive_user(db, setup_accounts, setup_roles_users): - """ - Set up one inactive user. - """ - from flexmeasures.data.models.user import User, Role - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - user_datastore.create_user( - username="inactive test user", - email="inactive@seita.nl", - password=hash_password("testtest"), - account_id=setup_accounts["Prosumer"].id, - active=False, - ) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py new file mode 100644 index 000000000..620ccf4fa --- /dev/null +++ b/flexmeasures/api/v3_0/__init__.py @@ -0,0 +1,13 @@ +from flask import Flask + +from flexmeasures.api.v3_0.sensors import SensorAPI +from flexmeasures.api.v3_0.users import UserAPI + + +def register_at(app: Flask): + """This can be used to register this blueprint together with other api-related things""" + + v3_0_api_prefix = "/api/v3_0" + + SensorAPI.register(app, route_prefix=v3_0_api_prefix) + UserAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py new file mode 100644 index 000000000..f327ce95e --- /dev/null +++ b/flexmeasures/api/v3_0/sensors.py @@ -0,0 +1,156 @@ +import json + +from flask_classful import FlaskView, route +from flask_json import as_json +from flask_security import auth_required +from timely_beliefs import BeliefsDataFrame +from webargs.flaskparser import use_args, use_kwargs + +from flexmeasures.api.common.schemas.sensor_data import ( + GetSensorDataSchema, + PostSensorDataSchema, +) +from flexmeasures.api.common.schemas.users import AccountIdField +from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures.auth.decorators import permission_required_for_context +from flexmeasures.data.models.user import Account +from flexmeasures.data.schemas.sensors import SensorSchema +from flexmeasures.data.services.sensors import get_sensors + +# Instantiate schemas outside of endpoint logic to minimize response time +get_sensor_schema = GetSensorDataSchema() +post_sensor_schema = PostSensorDataSchema() +sensors_schema = SensorSchema(many=True) + + +class SensorAPI(FlaskView): + + route_base = "/sensors" + trailing_slash = False + decorators = [auth_required()] + + @route("/", methods=["GET"]) + @use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + }, + location="query", + ) + @permission_required_for_context("read", arg_name="account") + @as_json + def index(self, account: Account): + """API endpoint to list all sensors of an account. + + .. :quickref: Sensor; Download sensor list + + This endpoint returns all accessible sensors. + Accessible sensors are sensors in the same account as the current user. + Only admins can use this endpoint to fetch sensors from a different account (by using the `account_id` query parameter). + + **Example response** + + An example of one sensor being returned: + + .. sourcecode:: json + + [ + { + "entity_address": "ea1.2021-01.io.flexmeasures.company:fm1.42", + "event_resolution": 15, + "generic_asset_id": 1, + "name": "Gas demand", + "timezone": "Europe/Amsterdam", + "unit": "m\u00b3/h" + } + ] + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + sensors = get_sensors(account_name=account.name) + return sensors_schema.dump(sensors), 200 + + @route("/data", methods=["POST"]) + @use_args( + post_sensor_schema, + location="json", + ) + def post_data(self, bdf: BeliefsDataFrame): + """ + Post sensor data to FlexMeasures. + + .. :quickref: Data; Upload sensor data + + **Example request** + + .. code-block:: json + + { + "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", + "values": [-11.28, -11.28, -11.28, -11.28], + "start": "2021-06-07T00:00:00+02:00", + "duration": "PT1H", + "unit": "m³/h" + } + + The above request posts four values for a duration of one hour, where the first + event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. + + The sensor is the one with ID=1. + The unit has to be convertible to the sensor's unit. + The resolution of the data has to match the sensor's required resolution, but + FlexMeasures will attempt to upsample lower resolutions. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + response, code = save_and_enqueue(bdf) + return response, code + + @route("/data", methods=["GET"]) + @use_args( + get_sensor_schema, + location="query", + ) + def get_data(self, response: dict): + """Get sensor data from FlexMeasures. + + .. :quickref: Data; Download sensor data + + **Example request** + + .. code-block:: json + + { + "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", + "start": "2021-06-07T00:00:00+02:00", + "duration": "PT1H", + "unit": "m³/h" + } + + The unit has to be convertible from the sensor's unit. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + return json.dumps(response) diff --git a/flexmeasures/api/v3_0/tests/__init__.py b/flexmeasures/api/v3_0/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py new file mode 100644 index 000000000..75ab132a7 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -0,0 +1,69 @@ +from datetime import timedelta + +import pytest +from flask_security import SQLAlchemySessionUserDatastore, hash_password + +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset +from flexmeasures.data.models.time_series import Sensor + + +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, setup_roles_users, setup_generic_assets): + """ + Set up data for API v3.0 tests. + """ + print("Setting up data for API v3.0 tests on %s" % db.engine) + add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + + +@pytest.fixture(scope="function") +def setup_api_fresh_test_data( + fresh_db, setup_roles_users_fresh_db, setup_generic_assets_fresh_db +): + """ + Set up fresh data for API dev tests. + """ + print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) + for sensor in Sensor.query.all(): + fresh_db.delete(sensor) + add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) + + +@pytest.fixture(scope="module") +def setup_inactive_user(db, setup_accounts, setup_roles_users): + """ + Set up one inactive user. + """ + from flexmeasures.data.models.user import User, Role + + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) + user_datastore.create_user( + username="inactive test user", + email="inactive@seita.nl", + password=hash_password("testtest"), + account_id=setup_accounts["Prosumer"].id, + active=False, + ) + + +def add_gas_sensor(db, test_supplier_user): + incineration_type = GenericAssetType( + name="waste incinerator", + ) + db.session.add(incineration_type) + db.session.flush() + incineration_asset = GenericAsset( + name="incineration line", + generic_asset_type=incineration_type, + account_id=test_supplier_user.account_id, + ) + db.session.add(incineration_asset) + db.session.flush() + gas_sensor = Sensor( + name="some gas sensor", + unit="m³/h", + event_resolution=timedelta(minutes=10), + generic_asset=incineration_asset, + ) + db.session.add(gas_sensor) + gas_sensor.owner = test_supplier_user.account diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py similarity index 81% rename from flexmeasures/api/v2_0/tests/test_api_v2_0_users.py rename to flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index f43f6db8e..0ae15b7a1 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -22,7 +22,7 @@ def test_get_users_bad_auth(client, use_auth): query = {"account_id": 2} get_users_response = client.get( - url_for("flexmeasures_api_v2_0.get_users"), headers=headers, query_string=query + url_for("UserAPI:index"), headers=headers, query_string=query ) print("Server responded with:\n%s" % get_users_response.data) if use_auth: @@ -43,7 +43,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): if include_inactive in (True, False): query["include_inactive"] = include_inactive get_users_response = client.get( - url_for("flexmeasures_api_v2_0.get_users"), query_string=query, headers=headers + url_for("UserAPI:index"), query_string=query, headers=headers ) print("Server responded with:\n%s" % get_users_response.json) assert get_users_response.status_code == 200 @@ -71,7 +71,7 @@ def test_get_one_user(client, requesting_user, status_code): headers["Authorization"] = get_auth_token(client, requesting_user, "testtest") get_user_response = client.get( - url_for("flexmeasures_api_v2_0.get_user", id=test_user2_id), + url_for("UserAPI:get", id=test_user2_id), headers=headers, ) print("Server responded with:\n%s" % get_user_response.data) @@ -89,7 +89,7 @@ def test_edit_user(client): admin_id = admin.id # without being the user themselves or an admin, the user cannot be edited user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=admin_id), + url_for("UserAPI:patch", id=admin_id), headers={ "content-type": "application/json", "Authorization": user2_auth_token, @@ -98,18 +98,20 @@ def test_edit_user(client): ) assert user_edit_response.status_code == 403 user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), + url_for("UserAPI:patch", id=user2_id), headers={"content-type": "application/json"}, json={}, ) assert user_edit_response.status_code == 401 - # admin can deactivate user2, other changes will be ignored - # (id is in the User schema of the API, but we ignore it) - headers = {"content-type": "application/json", "Authorization": admin_auth_token} + # admin can deactivate user2 + admin_headers = { + "content-type": "application/json", + "Authorization": admin_auth_token, + } user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), - headers=headers, - json={"active": False, "id": 888}, + url_for("UserAPI:patch", id=user2_id), + headers=admin_headers, + json={"active": False}, ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 200 @@ -118,29 +120,36 @@ def test_edit_user(client): assert user2.active is False assert user2.id == user2_id # admin can edit themselves but not sensitive fields - headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=admin_id), - headers=headers, + url_for("UserAPI:patch", id=admin_id), + headers=admin_headers, json={"active": False}, ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 403 -def test_edit_user_with_unexpected_fields(client): +@pytest.mark.parametrize( + "unexpected_fields", + [ + dict(password="I-should-not-be-sending-this"), # not part of the schema + dict(id=10), # id is a dump_only field + dict(account_id=10), # account_id is a dump_only field + ], +) +def test_edit_user_with_unexpected_fields(client, unexpected_fields: dict): """Sending unexpected fields (not in Schema) is an Unprocessable Entity error.""" with UserContext("test_prosumer_user_2@seita.nl") as user2: user2_id = user2.id with UserContext("test_admin_user@seita.nl") as admin: admin_auth_token = admin.get_auth_token() user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), + url_for("UserAPI:patch", id=user2_id), headers={ "content-type": "application/json", "Authorization": admin_auth_token, }, - json={"active": False, "password": "I-should-not-be-sending-this"}, + json={**{"active": False}, **unexpected_fields}, ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 422 diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py similarity index 95% rename from flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py rename to flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py index 74bde01f4..1603b8102 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py @@ -28,7 +28,7 @@ def test_user_reset_password(app, client, setup_inactive_user, sender): headers["Authorization"] = (get_auth_token(client, sender, "testtest"),) with app.mail.record_messages() as outbox: pwd_reset_response = client.patch( - url_for("flexmeasures_api_v2_0.reset_user_password", id=user2_id), + url_for("UserAPI:reset_user_password", id=user2_id), query_string={}, headers=headers, ) diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py similarity index 81% rename from flexmeasures/api/dev/tests/test_sensor_data.py rename to flexmeasures/api/v3_0/tests/test_sensor_data.py index 62d36add4..81180dba7 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -2,7 +2,7 @@ import pytest from flexmeasures.api.tests.utils import get_auth_token -from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor +from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @pytest.mark.parametrize("use_auth", [False, True]) @@ -14,14 +14,16 @@ def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): headers = {"content-type": "application/json"} if use_auth: # in this case, we successfully authenticate, - # but fail authorization (no admin or MDC role) + # but fail authorization (not member of the account in which the sensor lies) headers["Authorization"] = get_auth_token( client, "test_dummy_user_3@seita.nl", "testtest" ) + post_data = make_sensor_data_request_for_gas_sensor() post_data_response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), headers=headers, + json=post_data, ) print("Server responded with:\n%s" % post_data_response.data) if use_auth: @@ -51,9 +53,9 @@ def test_post_invalid_sensor_data( post_data = make_sensor_data_request_for_gas_sensor() post_data[request_field] = new_value # this guy is allowed to post sensorData - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) @@ -63,20 +65,21 @@ def test_post_invalid_sensor_data( def test_post_sensor_data_twice(client, setup_api_test_data): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") post_data = make_sensor_data_request_for_gas_sensor() # Check that 1st time posting the data succeeds response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) + print(response.json) assert response.status_code == 200 # Check that 2nd time posting the same data succeeds informatively response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) @@ -87,7 +90,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that replacing data fails informatively post_data["values"][0] = 100 response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py similarity index 88% rename from flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py rename to flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index ec3df96b1..d55923d8d 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -2,7 +2,7 @@ from flask import url_for from flexmeasures.api.tests.utils import get_auth_token -from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor +from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor from flexmeasures.data.models.time_series import TimedBelief, Sensor @@ -37,9 +37,9 @@ def test_post_sensor_data( print(f"BELIEFS BEFORE: {beliefs_before}") assert len(beliefs_before) == 0 - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( - url_for("post_sensor_data"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py new file mode 100644 index 000000000..c7477dd91 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -0,0 +1,23 @@ +from flexmeasures import Sensor + + +def make_sensor_data_request_for_gas_sensor( + num_values: int = 6, duration: str = "PT1H", unit: str = "m³" +) -> dict: + """Creates request to post sensor data for a gas sensor. + This particular gas sensor measures units of m³/h with a 10-minute resolution. + """ + sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() + message: dict = { + "type": "PostSensorDataRequest", + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "values": num_values * [-11.28], + "start": "2021-06-07T00:00:00+02:00", + "duration": duration, + "horizon": "PT0H", + "unit": unit, + } + if num_values == 1: + # flatten [] to + message["values"] = message["values"][0] + return message diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py new file mode 100644 index 000000000..c425ea7da --- /dev/null +++ b/flexmeasures/api/v3_0/users.py @@ -0,0 +1,233 @@ +from flask_classful import FlaskView, route +from marshmallow import fields +from sqlalchemy.exc import IntegrityError +from webargs.flaskparser import use_kwargs +from flask_security import current_user +from flask_security.recoverable import send_reset_password_instructions +from flask_json import as_json +from werkzeug.exceptions import Forbidden + +from flexmeasures.data.models.user import User as UserModel, Account +from flexmeasures.api.common.schemas.users import AccountIdField, UserIdField +from flexmeasures.data.schemas.users import UserSchema +from flexmeasures.data.services.users import ( + get_users, + set_random_password, + remove_cookie_and_token_access, +) +from flexmeasures.auth.decorators import permission_required_for_context +from flexmeasures.data import db + +""" +API endpoints to manage users. + +Both POST (to create) and DELETE are not accessible via the API, but as CLI functions. +""" + +# Instantiate schemas outside of endpoint logic to minimize response time +user_schema = UserSchema() +users_schema = UserSchema(many=True) +partial_user_schema = UserSchema(partial=True) + + +class UserAPI(FlaskView): + route_base = "/users" + trailing_slash = False + + @route("", methods=["GET"]) + @use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + "include_inactive": fields.Bool(load_default=False), + }, + location="query", + ) + @permission_required_for_context("read", arg_name="account") + @as_json + def index(self, account: Account, include_inactive: bool = False): + """API endpoint to list all users of an account. + + .. :quickref: User; Download user list + + This endpoint returns all accessible users. + By default, only active users are returned. + The `include_inactive` query parameter can be used to also fetch + inactive users. + Accessible users are users in the same account as the current user. + Only admins can use this endpoint to fetch users from a different account (by using the `account_id` query parameter). + + **Example response** + + An example of one user being returned: + + .. sourcecode:: json + + [ + { + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'account_id': 13, + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + ] + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + users = get_users(account_name=account.name, only_active=not include_inactive) + return users_schema.dump(users), 200 + + @route("/") + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("read", arg_name="user") + @as_json + def get(self, id: int, user: UserModel): + """API endpoint to get a user. + + .. :quickref: User; Get a user + + This endpoint gets a user. + Only admins or the user themselves can use this endpoint. + + **Example response** + + .. sourcecode:: json + + { + 'account_id': 1, + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + return user_schema.dump(user), 200 + + @route("/", methods=["PATCH"]) + @use_kwargs(partial_user_schema) + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("update", arg_name="user") + @as_json + def patch(self, id: int, user: UserModel, **user_data): + """API endpoint to patch user data. + + .. :quickref: User; Patch data for an existing user + + This endpoint sets data for an existing user. + Any subset of user fields can be sent. + Only the user themselves or admins are allowed to update its data, + while a non-admin can only edit a few of their own fields. + + Several fields are not allowed to be updated, e.g. id and account_id. + + **Example request** + + .. sourcecode:: json + + { + "active": false, + } + + **Example response** + + The following user fields are returned: + + .. sourcecode:: json + + { + 'account_id': 1, + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: UPDATED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + allowed_fields = [ + "email", + "username", + "active", + "timezone", + "flexmeasures_roles", + ] + for k, v in [(k, v) for k, v in user_data.items() if k in allowed_fields]: + if current_user.id == user.id and k in ("active", "flexmeasures_roles"): + raise Forbidden( + "Users who edit themselves cannot edit security-sensitive fields." + ) + setattr(user, k, v) + if k == "active" and v is False: + remove_cookie_and_token_access(user) + db.session.add(user) + try: + db.session.commit() + except IntegrityError as ie: + return ( + dict(message="Duplicate user already exists", detail=ie._message()), + 400, + ) + return user_schema.dump(user), 200 + + @route("//password-reset", methods=["PATCH"]) + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("update", arg_name="user") + @as_json + def reset_user_password(self, id: int, user: UserModel): + """API endpoint to reset the user's current password, cookies and auth tokens, and to email a password reset link to the user. + + .. :quickref: User; Password reset + + Reset the user's password, and send them instructions on how to reset the password. + This endpoint is useful from a security standpoint, in case of worries the password might be compromised. + It sets the current password to something random, invalidates cookies and auth tokens, + and also sends an email for resetting the password to the user. + + Users can reset their own passwords. Only admins can use this endpoint to reset passwords of other users. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + set_random_password(user) + remove_cookie_and_token_access(user) + send_reset_password_instructions(user) + + # commit only if sending instructions worked, as well + db.session.commit() diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 75af4c0ad..653c49fe7 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -8,15 +8,9 @@ roles_required as roles_required_fs, ) from werkzeug.local import LocalProxy -from werkzeug.exceptions import Forbidden, Unauthorized - -from flexmeasures.auth.policy import ( - ADMIN_ROLE, - PERMISSIONS, - AuthModelMixin, - user_has_admin_access, - user_matches_principals, -) +from werkzeug.exceptions import Forbidden + +from flexmeasures.auth.policy import ADMIN_ROLE, AuthModelMixin, check_access _security = LocalProxy(lambda: current_app.extensions["security"]) @@ -143,10 +137,6 @@ def view(resource_id: int, the_resource: Resource): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): - if permission not in PERMISSIONS: - raise Forbidden(f"Permission '{permission}' cannot be handled.") - if current_user.is_anonymous: - raise Unauthorized() # load & check context if arg_loader is not None: context: AuthModelMixin = arg_loader() @@ -158,26 +148,9 @@ def decorated_view(*args, **kwargs): context = kwargs[arg_name] else: context = args[0] - if context is None: - raise Forbidden( - f"Context needs {permission}-permission, but no context was passed." - ) - if not isinstance(context, AuthModelMixin): - raise Forbidden( - f"Context {context} needs {permission}-permission, but is no AuthModelMixin." - ) - # now check access, either with admin rights or principal(s) - acl = context.__acl__() - principals = acl.get(permission, tuple()) - current_app.logger.debug( - f"Looking for {permission}-permission on {context} ... Principals: {principals}" - ) - if not user_has_admin_access( - current_user, permission - ) and not user_matches_principals(current_user, principals): - raise Forbidden( - f"Authorization failure (accessing {context} to {permission}) ― cannot match {current_user} against {principals}!" - ) + + check_access(context, permission) + return fn(*args, **kwargs) return decorated_view diff --git a/flexmeasures/auth/policy.py b/flexmeasures/auth/policy.py index 37e030fbb..c7a0e6b0d 100644 --- a/flexmeasures/auth/policy.py +++ b/flexmeasures/auth/policy.py @@ -1,6 +1,8 @@ from typing import Dict, Union, Tuple, List from flask import current_app +from flask_security import current_user +from werkzeug.exceptions import Unauthorized, Forbidden PERMISSIONS = ["create-children", "read", "read-children", "update", "delete"] @@ -69,6 +71,45 @@ def __acl__(self) -> Dict[str, PRINCIPALS_TYPE]: return {} +def check_access(context: AuthModelMixin, permission: str): + """ + Check if current user can access this auth context if this permission + is required, either with admin rights or principal(s). + + Raises 401 or 403 otherwise. + """ + # check current user + if permission not in PERMISSIONS: + raise Forbidden(f"Permission '{permission}' cannot be handled.") + if current_user.is_anonymous: + raise Unauthorized() + + # check context + if context is None: + raise Forbidden( + f"Context needs {permission}-permission, but no context was passed." + ) + if not isinstance(context, AuthModelMixin): + raise Forbidden( + f"Context {context} needs {permission}-permission, but is no AuthModelMixin." + ) + + # look up principals + acl = context.__acl__() + principals: PRINCIPALS_TYPE = acl.get(permission, []) + current_app.logger.debug( + f"Looking for {permission}-permission on {context} ... Principals: {principals}" + ) + + # check access + if not user_has_admin_access( + current_user, permission + ) and not user_matches_principals(current_user, principals): + raise Forbidden( + f"Authorization failure (accessing {context} to {permission}) ― cannot match {current_user} against {principals}!" + ) + + def user_has_admin_access(user, permission: str) -> bool: if user.has_role(ADMIN_ROLE) or ( user.has_role(ADMIN_READER_ROLE) and permission == "read" diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index a3617477e..4f7ae95b1 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -68,11 +68,11 @@ class GenericAsset(db.Model, AuthModelMixin): def __acl__(self): """ All logged-in users can read if the asset is public. - Within same account, everyone can read and update. - Creation and deletion are left to account admins. + Within same account, everyone can create, read and update. + Deletion is left to account admins. """ return { - "create-children": (f"account:{self.account_id}", "role:account-admin"), + "create-children": f"account:{self.account_id}", "read": f"account:{self.account_id}" if self.account_id is not None else EVERY_LOGGED_IN_USER, diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index a509b840a..4faa6c35e 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -92,14 +92,11 @@ def __init__( def __acl__(self): """ All logged-in users can read if the sensor belongs to a public asset. - Within same account, everyone can read and update. - Creation and deletion are left to account admins. + Within same account, everyone can create, read and update. + Deletion is left to account admins. """ return { - "create-children": ( - f"account:{self.generic_asset.account_id}", - "role:account-admin", - ), + "create-children": f"account:{self.generic_asset.account_id}", "read": f"account:{self.generic_asset.account_id}" if self.generic_asset.account_id is not None else EVERY_LOGGED_IN_USER, diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index b30a42abb..136314ea8 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -1,12 +1,15 @@ from typing import Optional -from flask.cli import with_appcontext from marshmallow import validates, validates_schema, ValidationError, fields from flexmeasures.data import ma from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + FMValidationError, + MarshmallowClickMixin, + with_appcontext_if_needed, +) class GenericAssetSchema(ma.SQLAlchemySchema): @@ -96,7 +99,7 @@ class Meta: class GenericAssetIdField(MarshmallowClickMixin, fields.Int): """Field that deserializes to a GenericAsset and serializes back to an integer.""" - @with_appcontext + @with_appcontext_if_needed() def _deserialize(self, value, attr, obj, **kwargs) -> GenericAsset: """Turn a generic asset id into a GenericAsset.""" generic_asset = GenericAsset.query.get(value) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9e958b78b..6f96cbcae 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,10 +1,13 @@ -from flask.cli import with_appcontext from marshmallow import Schema, fields, validates, ValidationError from flexmeasures.data import ma from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + FMValidationError, + MarshmallowClickMixin, + with_appcontext_if_needed, +) from flexmeasures.utils.unit_utils import is_valid_unit @@ -26,6 +29,7 @@ class Meta: unit = ma.auto_field(required=True) timezone = ma.auto_field() event_resolution = fields.TimeDelta(required=True, precision="minutes") + entity_address = fields.String(dump_only=True) @validates("unit") def validate_unit(self, unit: str): @@ -55,14 +59,14 @@ class Meta: class SensorIdField(MarshmallowClickMixin, fields.Int): """Field that deserializes to a Sensor and serializes back to an integer.""" - @with_appcontext - def _deserialize(self, value, attr, obj, **kwargs) -> Sensor: + @with_appcontext_if_needed() + def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: """Turn a sensor id into a Sensor.""" sensor = Sensor.query.get(value) if sensor is None: raise FMValidationError(f"No sensor found with id {value}.") return sensor - def _serialize(self, value, attr, data, **kwargs): + def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: """Turn a Sensor into a sensor id.""" - return value.id + return sensor.id diff --git a/flexmeasures/data/schemas/users.py b/flexmeasures/data/schemas/users.py index dbec66c28..f1c175c95 100644 --- a/flexmeasures/data/schemas/users.py +++ b/flexmeasures/data/schemas/users.py @@ -19,10 +19,10 @@ def validate_timezone(self, timezone): if timezone not in all_timezones: raise ValidationError(f"Timezone {timezone} doesn't exist.") - id = ma.auto_field() + id = ma.auto_field(dump_only=True) email = ma.auto_field(required=True, validate=validate.Email) username = ma.auto_field(required=True) - account_id = ma.auto_field() + account_id = ma.auto_field(dump_only=True) active = ma.auto_field() timezone = ma.auto_field() flexmeasures_roles = ma.auto_field() diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index 37cbabbbc..a9056b47c 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -1,5 +1,7 @@ import click import marshmallow as ma +from click import get_current_context +from flask.cli import with_appcontext as with_cli_appcontext from marshmallow import ValidationError @@ -28,3 +30,14 @@ class FMValidationError(ValidationError): result = "Rejected" status = "UNPROCESSABLE_ENTITY" + + +def with_appcontext_if_needed(): + """Execute within the script's application context, in case there is one.""" + + def decorator(f): + if get_current_context(silent=True): + return with_cli_appcontext(f) + return f + + return decorator diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py new file mode 100644 index 000000000..1b09cf8b5 --- /dev/null +++ b/flexmeasures/data/services/sensors.py @@ -0,0 +1,26 @@ +from typing import Optional, List + +from werkzeug.exceptions import NotFound + +from flexmeasures import Sensor, Account +from flexmeasures.data.models.generic_assets import GenericAsset + + +def get_sensors( + account_name: Optional[str] = None, +) -> List[Sensor]: + """Return a list of Sensor objects. + + :param account_name: optionally, filter by account name. + """ + sensor_query = Sensor.query.join(GenericAsset).filter( + Sensor.generic_asset_id == GenericAsset.id + ) + + if account_name is not None: + account = Account.query.filter(Account.name == account_name).one_or_none() + if not account: + raise NotFound(f"There is no account named {account_name}!") + sensor_query = sensor_query.filter(GenericAsset.owner == account) + + return sensor_query.all() diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index ee68f6d80..0e03c6fd8 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -344,7 +344,6 @@ def delete_with_data(self, id: str): def _set_account(asset_form: NewAssetForm) -> Tuple[Optional[Account], Optional[str]]: """Set an account for the to-be-created asset. Return the account (if available) and an error message""" - account = None account_error = None if asset_form.account_id.data == -1: diff --git a/flexmeasures/ui/crud/users.py b/flexmeasures/ui/crud/users.py index 5e181e216..5b4785f2f 100644 --- a/flexmeasures/ui/crud/users.py +++ b/flexmeasures/ui/crud/users.py @@ -84,7 +84,7 @@ def index(self): for account in Account.query.all(): get_users_response = InternalApi().get( url_for( - "flexmeasures_api_v2_0.get_users", + "UserAPI:index", account_id=account.id, include_inactive=include_inactive, ) @@ -100,9 +100,7 @@ def index(self): @roles_required(ADMIN_ROLE) def get(self, id: str): """GET from /users/""" - get_user_response = InternalApi().get( - url_for("flexmeasures_api_v2_0.get_user", id=id) - ) + get_user_response = InternalApi().get(url_for("UserAPI:get", id=id)) user: User = process_internal_api_response( get_user_response.json(), make_obj=True ) @@ -119,7 +117,7 @@ def toggle_active(self, id: str): """Toggle activation status via /users/toggle_active/""" user: User = get_user(id) user_response = InternalApi().patch( - url_for("flexmeasures_api_v2_0.patch_user", id=id), + url_for("UserAPI:patch", id=id), args={"active": not user.active}, ) patched_user: User = process_internal_api_response( @@ -138,7 +136,7 @@ def reset_password_for(self, id: str): and send instructions on how to reset.""" user: User = get_user(id) InternalApi().patch( - url_for("flexmeasures_api_v2_0.reset_user_password", id=id), + url_for("UserAPI:reset_user_password", id=id), ) return render_user( user, diff --git a/flexmeasures/ui/tests/test_user_crud.py b/flexmeasures/ui/tests/test_user_crud.py index 71ed4543f..c4506fa5f 100644 --- a/flexmeasures/ui/tests/test_user_crud.py +++ b/flexmeasures/ui/tests/test_user_crud.py @@ -23,7 +23,7 @@ def test_user_crud_as_non_admin(client, as_prosumer_user1, view): def test_user_list(client, as_admin, requests_mock): requests_mock.get( - "http://localhost//api/v2_0/users", + "http://localhost//api/v3_0/users", status_code=200, json=mock_user_response(multiple=True), ) @@ -37,7 +37,7 @@ def test_user_list(client, as_admin, requests_mock): def test_user_page(client, as_admin, requests_mock): mock_user = mock_user_response(as_list=False) requests_mock.get( - "http://localhost//api/v2_0/user/2", status_code=200, json=mock_user + "http://localhost//api/v3_0/users/2", status_code=200, json=mock_user ) requests_mock.get( "http://localhost//api/v2_0/assets", @@ -55,7 +55,7 @@ def test_deactivate_user(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v2_0/user/{user2.id}", + f"http://localhost//api/v3_0/users/{user2.id}", status_code=200, json={"active": False}, ) @@ -72,7 +72,7 @@ def test_reset_password(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v2_0/user/{user2.id}/password-reset", + f"http://localhost//api/v3_0/users/{user2.id}/password-reset", status_code=200, ) user_page = client.get( diff --git a/flexmeasures/ui/tests/utils.py b/flexmeasures/ui/tests/utils.py index 1f2e0c747..9d2aa8953 100644 --- a/flexmeasures/ui/tests/utils.py +++ b/flexmeasures/ui/tests/utils.py @@ -1,4 +1,5 @@ import copy +from typing import List, Union from flask import url_for @@ -22,7 +23,7 @@ def mock_asset_response( account_id: int = 1, as_list: bool = True, multiple: bool = False, -) -> dict: +) -> Union[dict, List[dict]]: asset = dict( id=asset_id, name="TestAsset", @@ -48,7 +49,7 @@ def mock_user_response( active: bool = True, as_list: bool = True, multiple: bool = False, -) -> dict: +) -> Union[dict, List[dict]]: user = dict( id=user_id, username=username, @@ -64,7 +65,7 @@ def mock_user_response( user2 = copy.deepcopy(user) user2["id"] = 2 user2["username"] = "Bert" - user2["email"] = ("bert@seita.nl",) + user2["email"] = "bert@seita.nl" user_list.append(user2) return user_list return user diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 96e69608c..cfeeaa791 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd import pint +import timely_beliefs as tb # Edit constants template to stop using h to represent planck_constant constants_template = ( @@ -151,10 +152,14 @@ def units_are_convertible( def is_power_unit(unit: str) -> bool: """For example: - >>> is_power_unit("kW") # True - >>> is_power_unit("°C") # False - >>> is_power_unit("kWh") # False - >>> is_power_unit("EUR/MWh") # False + >>> is_power_unit("kW") + True + >>> is_power_unit("°C") + False + >>> is_power_unit("kWh") + False + >>> is_power_unit("EUR/MWh") + False """ if not is_valid_unit(unit): return False @@ -163,18 +168,42 @@ def is_power_unit(unit: str) -> bool: def is_energy_unit(unit: str) -> bool: """For example: - >>> is_energy_unit("kW") # False - >>> is_energy_unit("°C") # False - >>> is_energy_unit("kWh") # True - >>> is_energy_unit("EUR/MWh") # False + >>> is_energy_unit("kW") + False + >>> is_energy_unit("°C") + False + >>> is_energy_unit("kWh") + True + >>> is_energy_unit("EUR/MWh") + False """ if not is_valid_unit(unit): return False return ur.Quantity(unit).dimensionality == ur.Quantity("Wh").dimensionality +def is_energy_price_unit(unit: str) -> bool: + """For example: + >>> is_energy_price_unit("EUR/MWh") + True + >>> is_energy_price_unit("KRW/MWh") + True + >>> is_energy_price_unit("KRW/MW") + False + >>> is_energy_price_unit("beans/MW") + False + """ + if ( + unit[:3] in [str(c) for c in list_all_currencies()] + and unit[3] == "/" + and is_energy_unit(unit[4:]) + ): + return True + return False + + def convert_units( - data: Union[pd.Series, List[Union[int, float]], int, float], + data: Union[tb.BeliefsSeries, pd.Series, List[Union[int, float]], int, float], from_unit: str, to_unit: str, event_resolution: Optional[timedelta] = None, @@ -216,6 +245,8 @@ def convert_units( ) else: # Catch multiplicative conversions that use the resolution, like "kWh/15min" to "kW" + if event_resolution is None and isinstance(data, tb.BeliefsSeries): + event_resolution = data.event_resolution multiplier = determine_unit_conversion_multiplier( from_unit, to_unit, event_resolution )