From f6f1b918a44eba0eded0fc1d0fffa9919a64dd8c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 7 Mar 2022 15:28:03 +0100 Subject: [PATCH 01/99] Move sensor data API from dev to v2 and use flask-classful Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/__init__.py | 34 ------------ flexmeasures/api/dev/sensor_data.py | 28 ---------- .../api/dev/tests/test_sensor_data.py | 10 ++-- .../dev/tests/test_sensor_data_fresh_db.py | 2 +- flexmeasures/api/v2_0/__init__.py | 7 ++- .../api/v2_0/implementations/__init__.py | 2 +- .../api/v2_0/implementations/sensor_data.py | 55 +++++++++++++++++++ 7 files changed, 68 insertions(+), 70 deletions(-) delete mode 100644 flexmeasures/api/dev/sensor_data.py create mode 100644 flexmeasures/api/v2_0/implementations/sensor_data.py diff --git a/flexmeasures/api/dev/__init__.py b/flexmeasures/api/dev/__init__.py index d4f90bf9e..f82da36ce 100644 --- a/flexmeasures/api/dev/__init__.py +++ b/flexmeasures/api/dev/__init__.py @@ -1,7 +1,4 @@ from flask import Flask -from flask_security import auth_token_required - -from flexmeasures.auth.decorators import account_roles_accepted def register_at(app: Flask): @@ -9,39 +6,8 @@ def register_at(app: Flask): from flexmeasures.api.dev.sensors import SensorAPI from flexmeasures.api.dev.assets import AssetAPI - from flexmeasures.api.dev.sensor_data import post_data as post_sensor_data_impl dev_api_prefix = "/api/dev" SensorAPI.register(app, route_prefix=dev_api_prefix) AssetAPI.register(app, route_prefix=dev_api_prefix) - - @app.route(f"{dev_api_prefix}/sensorData", methods=["POST"]) - @auth_token_required - @account_roles_accepted("MDC", "Prosumer") - def post_sensor_data(): - """ - Post sensor data to FlexMeasures. - - For example: - - { - "type": "PostSensorDataRequest", - "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", - "values": [-11.28, -11.28, -11.28, -11.28], - "start": "2021-06-07T00:00:00+02:00", - "duration": "PT1H", - "unit": "m³/h", - } - - The above request posts four values for a duration of one hour, where the first - event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. - - The sensor is the one with ID=1. - The unit has to match the sensor's required unit. - The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. - """ - return post_sensor_data_impl() - - # TODO: add GET /sensorData diff --git a/flexmeasures/api/dev/sensor_data.py b/flexmeasures/api/dev/sensor_data.py deleted file mode 100644 index a8dd758f6..000000000 --- a/flexmeasures/api/dev/sensor_data.py +++ /dev/null @@ -1,28 +0,0 @@ -from webargs.flaskparser import use_args - -from flexmeasures.api.common.schemas.sensor_data import SensorDataSchema -from flexmeasures.api.common.utils.api_utils import save_and_enqueue - - -@use_args( - SensorDataSchema(), - location="json", -) -def post_data(sensor_data): - """POST to /sensorData - - Experimental dev feature which uses timely-beliefs - to create and save the data structure. - """ - beliefs = SensorDataSchema.load_bdf(sensor_data) - response, code = save_and_enqueue(beliefs) - response.update(type="PostSensorDataResponse") - return response, code - - -def get_data(): - """GET from /sensorData""" - # - use data.models.time_series.Sensor::search_beliefs() - might need to add a belief_horizon parameter - # - create the serialize method on the schema, to turn the resulting BeliefsDataFrame - # to the JSON the API should respond with. - pass diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/dev/tests/test_sensor_data.py index 62d36add4..ff660d2fc 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data.py +++ b/flexmeasures/api/dev/tests/test_sensor_data.py @@ -20,7 +20,7 @@ def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): ) post_data_response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), headers=headers, ) print("Server responded with:\n%s" % post_data_response.data) @@ -53,7 +53,7 @@ def test_post_invalid_sensor_data( # this guy is allowed to post sensorData auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), json=post_data, headers={"Authorization": auth_token}, ) @@ -68,7 +68,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that 1st time posting the data succeeds response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), json=post_data, headers={"Authorization": auth_token}, ) @@ -76,7 +76,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that 2nd time posting the same data succeeds informatively response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), json=post_data, headers={"Authorization": auth_token}, ) @@ -87,7 +87,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that replacing data fails informatively post_data["values"][0] = 100 response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py index ec3df96b1..5ec661187 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py @@ -39,7 +39,7 @@ def test_post_sensor_data( auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") response = client.post( - url_for("post_sensor_data"), + url_for("SensorDataAPI:post"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index 06f4d23b9..25c2ef48f 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -6,6 +6,11 @@ def register_at(app: Flask): """This can be used to register this blueprint together with other api-related things""" + from flexmeasures.api.v2_0.implementations.sensor_data import SensorDataAPI import flexmeasures.api.v2_0.routes # noqa: F401 this is necessary to load the endpoints - app.register_blueprint(flexmeasures_api, url_prefix="/api/v2_0") + v2_0_api_prefix = "/api/v2_0" + + app.register_blueprint(flexmeasures_api, url_prefix=v2_0_api_prefix) + + SensorDataAPI.register(app, route_prefix=v2_0_api_prefix) diff --git a/flexmeasures/api/v2_0/implementations/__init__.py b/flexmeasures/api/v2_0/implementations/__init__.py index c27fc08eb..e35323de0 100644 --- a/flexmeasures/api/v2_0/implementations/__init__.py +++ b/flexmeasures/api/v2_0/implementations/__init__.py @@ -1 +1 @@ -from . import assets, sensors, users # noqa F401 +from . import assets, sensor_data, sensors, users # noqa F401 diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py new file mode 100644 index 000000000..71b30602b --- /dev/null +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -0,0 +1,55 @@ +from flask_classful import FlaskView, route +from flask_security import auth_token_required +from webargs.flaskparser import use_args + +from flexmeasures.api.common.schemas.sensor_data import SensorDataSchema +from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures.auth.decorators import account_roles_accepted + + +class SensorDataAPI(FlaskView): + + route_base = "/sensorData" + + @auth_token_required + @account_roles_accepted("MDC", "Prosumer") + @use_args( + SensorDataSchema(), + location="json", + ) + @route("/", methods=["POST"]) + def post(self, sensor_data): + """ + Post sensor data to FlexMeasures. + + For example: + + { + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", + "values": [-11.28, -11.28, -11.28, -11.28], + "start": "2021-06-07T00:00:00+02:00", + "duration": "PT1H", + "unit": "m³/h", + } + + The above request posts four values for a duration of one hour, where the first + event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. + + The sensor is the one with ID=1. + The unit has to match the sensor's required unit. + The resolution of the data has to match the sensor's required resolution, but + FlexMeasures will attempt to upsample lower resolutions. + """ + beliefs = SensorDataSchema.load_bdf(sensor_data) + response, code = save_and_enqueue(beliefs) + response.update(type="PostSensorDataResponse") + return response, code + + @route("/", methods=["GET"]) + def get(self): + """GET from /sensorData""" + # - use data.models.time_series.Sensor::search_beliefs() - might need to add a belief_horizon parameter + # - create the serialize method on the schema, to turn the resulting BeliefsDataFrame + # to the JSON the API should respond with. + pass From f92d18254b8731b2e4ee9071cd0764cce54e8518 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 09:27:11 +0100 Subject: [PATCH 02/99] Publish dev API Signed-off-by: F.N. Claessen --- documentation/api/dev.rst | 22 ++++++++++++++++++++++ documentation/index.rst | 1 + 2 files changed, 23 insertions(+) create mode 100644 documentation/api/dev.rst diff --git a/documentation/api/dev.rst b/documentation/api/dev.rst new file mode 100644 index 000000000..19cd61ef9 --- /dev/null +++ b/documentation/api/dev.rst @@ -0,0 +1,22 @@ +.. _dev: + +Developer API +============= + +These endpoints are still under development and are subject to change in new releases. + +Summary +------- + +.. qrefflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.dev.assets, flexmeasures.api.dev.sensors + :order: path + :include-empty-docstring: + +API Details +----------- + +.. autoflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.dev.assets, flexmeasures.api.dev.sensors + :order: path + :include-empty-docstring: diff --git a/documentation/index.rst b/documentation/index.rst index d0854e3cd..86aa47fec 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -133,6 +133,7 @@ The platform operator of FlexMeasures can be an Aggregator. api/v1_2 api/v1_1 api/v1 + api/dev api/change_log .. toctree:: From 84f4169273cf746aca000af6b2e9070b5f8bbfdb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 09:35:07 +0100 Subject: [PATCH 03/99] Add quickrefs Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/assets.py | 20 ++++++++++++++++---- flexmeasures/api/dev/sensors.py | 12 ++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/dev/assets.py b/flexmeasures/api/dev/assets.py index 94245bb3d..3b840cd93 100644 --- a/flexmeasures/api/dev/assets.py +++ b/flexmeasures/api/dev/assets.py @@ -45,7 +45,10 @@ def index(self, account: Account): ) @use_args(AssetSchema()) def post(self, asset_data): - """Create new asset""" + """Create new asset. + + .. :quickref: Asset; Create a new asset + """ asset = AssetModel(**asset_data) db.session.add(asset) db.session.commit() @@ -56,7 +59,10 @@ def post(self, asset_data): @permission_required_for_context("read", arg_name="asset") @as_json def fetch_one(self, id, asset): - """Fetch a given asset""" + """Fetch a given asset. + + .. :quickref: Asset; Get an asset + """ return asset_schema.dump(asset), 200 @route("/", methods=["PATCH"]) @@ -65,7 +71,10 @@ def fetch_one(self, id, asset): @permission_required_for_context("update", arg_name="db_asset") @as_json def patch(self, asset_data, id, db_asset): - """Update an asset given its identifier""" + """Update an asset given its identifier. + + .. :quickref: Asset; Update an asset + """ ignored_fields = ["id", "account_id"] for k, v in [(k, v) for k, v in asset_data.items() if k not in ignored_fields]: setattr(db_asset, k, v) @@ -78,7 +87,10 @@ def patch(self, asset_data, id, db_asset): @permission_required_for_context("delete", arg_name="asset") @as_json def delete(self, id, asset): - """Delete an asset given its identifier""" + """Delete an asset given its identifier. + + .. :quickref: Asset; Delete an asset + """ asset_name = asset.name db.session.delete(asset) db.session.commit() diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 2d19edf3f..d15601903 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -35,7 +35,10 @@ class SensorAPI(FlaskView): location="query", ) def get_chart(self, id, **kwargs): - """GET from /sensor//chart""" + """GET from /sensor//chart + + .. :quickref: Chart; Download a chart with time series + """ sensor = get_sensor_or_abort(id) return json.dumps(sensor.chart(**kwargs)) @@ -53,6 +56,8 @@ def get_chart(self, id, **kwargs): def get_chart_data(self, id, **kwargs): """GET from /sensor//chart_data + .. :quickref: Chart; Download time series for use in charts + Data for use in charts (in case you have the chart specs already). """ sensor = get_sensor_or_abort(id) @@ -60,7 +65,10 @@ def get_chart_data(self, id, **kwargs): @auth_required() def get(self, id: int): - """GET from /sensor/""" + """GET from /sensor/ + + .. :quickref: Chart; Download sensor attributes for use in charts + """ sensor = get_sensor_or_abort(id) attributes = ["name", "timezone", "timerange"] return {attr: getattr(sensor, attr) for attr in attributes} From 9eb72993785f14e724be65e68885011008587354 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 10:01:30 +0100 Subject: [PATCH 04/99] Add type annotations Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index d15601903..d7477c888 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -34,7 +34,7 @@ class SensorAPI(FlaskView): }, location="query", ) - def get_chart(self, id, **kwargs): + def get_chart(self, id: int, **kwargs): """GET from /sensor//chart .. :quickref: Chart; Download a chart with time series @@ -53,7 +53,7 @@ def get_chart(self, id, **kwargs): }, location="query", ) - def get_chart_data(self, id, **kwargs): + def get_chart_data(self, id: int, **kwargs): """GET from /sensor//chart_data .. :quickref: Chart; Download time series for use in charts From ea7a77d9b7607cf97bf2a547eeef91dcd22fd390 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 10:58:18 +0100 Subject: [PATCH 05/99] Refactor use of auth_required in SensorAPI Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/sensors.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index d7477c888..f17e01b2e 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -18,8 +18,8 @@ class SensorAPI(FlaskView): """ route_base = "/sensor" + decorators = [auth_required()] - @auth_required() @route("//chart/") @use_kwargs( { @@ -42,7 +42,6 @@ def get_chart(self, id: int, **kwargs): sensor = get_sensor_or_abort(id) return json.dumps(sensor.chart(**kwargs)) - @auth_required() @route("//chart_data/") @use_kwargs( { @@ -63,7 +62,6 @@ def get_chart_data(self, id: int, **kwargs): sensor = get_sensor_or_abort(id) return sensor.search_beliefs(as_json=True, **kwargs) - @auth_required() def get(self, id: int): """GET from /sensor/ From 9d2f329fc3b84a7d368451ef924b8030419fff77 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:05:59 +0100 Subject: [PATCH 06/99] Add type annotations Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/assets.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/dev/assets.py b/flexmeasures/api/dev/assets.py index 3b840cd93..1bf41ed88 100644 --- a/flexmeasures/api/dev/assets.py +++ b/flexmeasures/api/dev/assets.py @@ -6,7 +6,7 @@ from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data import db from flexmeasures.data.models.user import Account -from flexmeasures.data.models.generic_assets import GenericAsset as AssetModel +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField @@ -44,12 +44,12 @@ def index(self, account: Account): "create-children", arg_loader=AccountIdField.load_current ) @use_args(AssetSchema()) - def post(self, asset_data): + def post(self, asset_data: dict): """Create new asset. .. :quickref: Asset; Create a new asset """ - asset = AssetModel(**asset_data) + asset = GenericAsset(**asset_data) db.session.add(asset) db.session.commit() return asset_schema.dump(asset), 201 @@ -70,7 +70,7 @@ def fetch_one(self, id, asset): @use_kwargs({"db_asset": AssetIdField(data_key="id")}, location="path") @permission_required_for_context("update", arg_name="db_asset") @as_json - def patch(self, asset_data, id, db_asset): + def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): """Update an asset given its identifier. .. :quickref: Asset; Update an asset @@ -86,7 +86,7 @@ def patch(self, asset_data, id, db_asset): @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") @permission_required_for_context("delete", arg_name="asset") @as_json - def delete(self, id, asset): + def delete(self, id: int, asset: GenericAsset): """Delete an asset given its identifier. .. :quickref: Asset; Delete an asset From 25ba2de24677c1a4affa5c43c18195f27735940a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:07:59 +0100 Subject: [PATCH 07/99] Add more quickrefs Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/assets.py | 5 ++++- flexmeasures/api/v2_0/implementations/sensor_data.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/dev/assets.py b/flexmeasures/api/dev/assets.py index 1bf41ed88..1b2650ed7 100644 --- a/flexmeasures/api/dev/assets.py +++ b/flexmeasures/api/dev/assets.py @@ -36,7 +36,10 @@ class AssetAPI(FlaskView): @permission_required_for_context("read", arg_name="account") @as_json def index(self, account: Account): - """List all assets owned by a certain account.""" + """List all assets owned by a certain account. + + .. :quickref: Asset; Download asset list + """ return assets_schema.dump(account.generic_assets), 200 @route("/", methods=["POST"]) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 71b30602b..4f58cc149 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -22,6 +22,8 @@ def post(self, sensor_data): """ Post sensor data to FlexMeasures. + .. :quickref: Data; Upload sensor data + For example: { @@ -48,7 +50,10 @@ def post(self, sensor_data): @route("/", methods=["GET"]) def get(self): - """GET from /sensorData""" + """Get sensor data from FlexMeasures. + + .. :quickref: Data; Download sensor data + """ # - use data.models.time_series.Sensor::search_beliefs() - might need to add a belief_horizon parameter # - create the serialize method on the schema, to turn the resulting BeliefsDataFrame # to the JSON the API should respond with. From 01261699d60338bf9db5e6956725d09559745e64 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:11:36 +0100 Subject: [PATCH 08/99] Indentation Signed-off-by: F.N. Claessen --- .../api/v2_0/implementations/sensor_data.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 4f58cc149..8d2b36ff2 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -26,14 +26,14 @@ def post(self, sensor_data): For example: - { - "type": "PostSensorDataRequest", - "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", - "values": [-11.28, -11.28, -11.28, -11.28], - "start": "2021-06-07T00:00:00+02:00", - "duration": "PT1H", - "unit": "m³/h", - } + { + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", + "values": [-11.28, -11.28, -11.28, -11.28], + "start": "2021-06-07T00:00:00+02:00", + "duration": "PT1H", + "unit": "m³/h", + } The above request posts four values for a duration of one hour, where the first event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. From 7ce3f1444de0e193663b674cc14a6b91eb485f39 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:11:59 +0100 Subject: [PATCH 09/99] valid JSON, rather than a python dict Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 8d2b36ff2..b35158e73 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -24,7 +24,9 @@ def post(self, sensor_data): .. :quickref: Data; Upload sensor data - For example: + **Example request** + + .. code-block:: json { "type": "PostSensorDataRequest", @@ -32,7 +34,7 @@ def post(self, sensor_data): "values": [-11.28, -11.28, -11.28, -11.28], "start": "2021-06-07T00:00:00+02:00", "duration": "PT1H", - "unit": "m³/h", + "unit": "m³/h" } The above request posts four values for a duration of one hour, where the first From cb205ba868e73e58871abe656d267eec400711f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:30:15 +0100 Subject: [PATCH 10/99] Move import to its preferred position Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index 25c2ef48f..00b6597c3 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -1,12 +1,13 @@ from flask import Flask, Blueprint +from flexmeasures.api.v2_0.implementations.sensor_data import SensorDataAPI + flexmeasures_api = Blueprint("flexmeasures_api_v2_0", __name__) def register_at(app: Flask): """This can be used to register this blueprint together with other api-related things""" - from flexmeasures.api.v2_0.implementations.sensor_data import SensorDataAPI import flexmeasures.api.v2_0.routes # noqa: F401 this is necessary to load the endpoints v2_0_api_prefix = "/api/v2_0" From 841f1d39e1dedc5e960326a4b282ffee646d532f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 11:30:59 +0100 Subject: [PATCH 11/99] Rename and add type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index a735e2de8..a0add10fb 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -56,13 +56,13 @@ class SensorIdField(fields.Int, MarshmallowClickMixin): """Field that deserializes to a Sensor and serializes back to an integer.""" @with_appcontext - def _deserialize(self, value, attr, obj, **kwargs) -> Sensor: + def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: """Turn a sensor id into a Sensor.""" sensor = Sensor.query.get(value) if sensor is None: raise FMValidationError(f"No sensor found with id {value}.") return sensor - def _serialize(self, value, attr, data, **kwargs): + def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: """Turn a Sensor into a sensor id.""" - return value.id + return sensor.id From f126561dfe4cb4d5e8bdc9921f1402746f052a46 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 12:47:34 +0100 Subject: [PATCH 12/99] Allow AssetIdField and SensorIdField to be used for API requests, too, besides for CLI scripts Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 9 ++++++--- flexmeasures/data/schemas/sensors.py | 9 ++++++--- flexmeasures/data/schemas/utils.py | 13 +++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 1eafe513a..a8ef0058c 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -1,12 +1,15 @@ from typing import Optional -from flask.cli import with_appcontext from marshmallow import validates, validates_schema, ValidationError, fields from flexmeasures.data import ma from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + FMValidationError, + MarshmallowClickMixin, + with_appcontext, +) class GenericAssetSchema(ma.SQLAlchemySchema): @@ -96,7 +99,7 @@ class Meta: class GenericAssetIdField(fields.Int, MarshmallowClickMixin): """Field that deserializes to a GenericAsset and serializes back to an integer.""" - @with_appcontext + @with_appcontext() def _deserialize(self, value, attr, obj, **kwargs) -> GenericAsset: """Turn a generic asset id into a GenericAsset.""" generic_asset = GenericAsset.query.get(value) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index a0add10fb..1eca8b4bd 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,10 +1,13 @@ -from flask.cli import with_appcontext from marshmallow import Schema, fields, validates, ValidationError from flexmeasures.data import ma from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.schemas.utils import FMValidationError, MarshmallowClickMixin +from flexmeasures.data.schemas.utils import ( + FMValidationError, + MarshmallowClickMixin, + with_appcontext, +) from flexmeasures.utils.unit_utils import is_valid_unit @@ -55,7 +58,7 @@ class Meta: class SensorIdField(fields.Int, MarshmallowClickMixin): """Field that deserializes to a Sensor and serializes back to an integer.""" - @with_appcontext + @with_appcontext() def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: """Turn a sensor id into a Sensor.""" sensor = Sensor.query.get(value) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index 42276d8e5..a7c208e3a 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -1,5 +1,7 @@ import click import marshmallow as ma +from click import get_current_context +from flask.cli import with_appcontext as with_cli_appcontext from marshmallow import ValidationError @@ -24,3 +26,14 @@ class FMValidationError(ValidationError): result = "Rejected" status = "UNPROCESSABLE_ENTITY" + + +def with_appcontext(): + """Execute within the script's application context, in case there is one.""" + + def decorator(f): + if get_current_context(silent=True): + return with_cli_appcontext(f) + return f + + return decorator From 2e3bf11128b4682d0c9e9f9f62414d53f4533f5f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 15:34:34 +0100 Subject: [PATCH 13/99] Update tutorial for posting data Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index c15cd4e9c..201fec5e4 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -26,23 +26,27 @@ Prerequisites .. note:: To address assets and sensors, these tutorials assume entity addresses valid in the namespace ``fm1``. See :ref:`api_introduction` for more explanations. -Posting price data ------------------- +Posting sensor data +------------------- -Price data (both observations and forecasts) can be posted to `POST /api/v2_0/postPriceData <../api/v2_0.html#post--api-v2_0-postPriceData>`_. The URL might look like this: +Sensor data (both observations and forecasts) can be posted to `POST /api/v2_0/postSensorData <../api/v2_0.html#post--api-v2_0-postSensorData>`_. +This endpoint represents the basic method of getting time series data into FlexMeasures via API. +It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that happened in the past or in the future. +Some examples: readings from electricity meters, gas meters, temperature sensors and pressure sensors, the availability of parking spots, and price forecasts. +The exact URL will depend on your domain name, and will look approximately like this: .. code-block:: html - https://company.flexmeasures.io/api//postPriceData + https://company.flexmeasures.io/api//postSensorData -This example "PostPriceDataRequest" message posts prices for hourly intervals between midnight and midnight the next day +This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. The ``prior`` indicates that the prices were published at 3pm on December 31st 2014 (i.e. the clearing time of the KPX day-ahead market, which is at 3 PM on the previous day ― see below for a deeper explanation). .. code-block:: json { - "type": "PostPriceDataRequest", + "type": "PostSensorDataRequest", "market": "ea1.2021-01.io.flexmeasures.company:fm1.16", "values": [ 52.37, @@ -78,6 +82,7 @@ The ``prior`` indicates that the prices were published at 3pm on December 31st 2 Note how the resolution of the data comes out at 60 minutes when you divide the duration by the number of data points. If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match or, if that is not possible, complain. +Likewise, if the data unit does not match the sensor’s unit, FlexMeasures will attempt to convert the data or, if that is not possible, complain. Posting power data From e6db62f3a6e978613462b38bcccb55b6b870772b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 16:42:41 +0100 Subject: [PATCH 14/99] Implement [GET] sensorData Signed-off-by: F.N. Claessen --- .../api/v2_0/implementations/sensor_data.py | 65 ++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index b35158e73..2e9f867ad 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -1,17 +1,27 @@ +import json + +from isodate import datetime_isoformat, duration_isoformat +import pandas as pd from flask_classful import FlaskView, route -from flask_security import auth_token_required +from flask_security import login_required from webargs.flaskparser import use_args -from flexmeasures.api.common.schemas.sensor_data import SensorDataSchema +from flexmeasures.api.common.schemas.sensor_data import ( + SensorDataSchema, + SensorDataDescriptionSchema, +) from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures import Sensor from flexmeasures.auth.decorators import account_roles_accepted +from flexmeasures.data.services.time_series import simplify_index +from flexmeasures.utils.unit_utils import convert_units class SensorDataAPI(FlaskView): route_base = "/sensorData" + decorators = [login_required] - @auth_token_required @account_roles_accepted("MDC", "Prosumer") @use_args( SensorDataSchema(), @@ -51,12 +61,51 @@ def post(self, sensor_data): return response, code @route("/", methods=["GET"]) - def get(self): + @use_args( + SensorDataDescriptionSchema(), + location="query", + ) + def get(self, sensor_data_description): """Get sensor data from FlexMeasures. .. :quickref: Data; Download sensor data """ - # - use data.models.time_series.Sensor::search_beliefs() - might need to add a belief_horizon parameter - # - create the serialize method on the schema, to turn the resulting BeliefsDataFrame - # to the JSON the API should respond with. - pass + # todo: respect passed horizon and prior + # todo: move some of the below logic to the dump_bdf (serialize) method on the schema + sensor: Sensor = sensor_data_description["sensor"] + start = sensor_data_description["start"] + duration = sensor_data_description["duration"] + end = sensor_data_description["start"] + duration + unit = sensor_data_description["unit"] + + df = simplify_index( + sensor.search_beliefs( + event_starts_after=start, + event_ends_before=end, + one_deterministic_belief_per_event=True, + as_json=False, + ) + ) + + # Convert to desired time range + index = pd.date_range( + start=start, end=end, freq=sensor.event_resolution, closed="left" + ) + df = df.reindex(index).reset_index() + + # Convert to desired unit + values = convert_units( + df["event_value"], + from_unit=sensor.unit, + to_unit=unit, + ).to_list() + + response = dict( + values=values, + start=datetime_isoformat(start), + duration=duration_isoformat(duration), + unit=unit, + ) + + response.update(type="GetSensorDataResponse") + return json.dumps(response) From de46436df8f87c6b87ade17d8f151e6dbd9913a6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 16:43:38 +0100 Subject: [PATCH 15/99] Add example GET request Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 2e9f867ad..130a4e50d 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -69,6 +69,18 @@ def get(self, sensor_data_description): """Get sensor data from FlexMeasures. .. :quickref: Data; Download sensor data + + **Example request** + + .. code-block:: json + + { + "type": "GetSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", + "start": "2021-06-07T00:00:00+02:00", + "duration": "PT1H", + "unit": "m³/h" + } """ # todo: respect passed horizon and prior # todo: move some of the below logic to the dump_bdf (serialize) method on the schema From 2ab771b09a1c2233db241df82f547f722b409eef Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 16:45:24 +0100 Subject: [PATCH 16/99] Update docstring for unit conversion Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 130a4e50d..a8826486a 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -51,7 +51,7 @@ def post(self, sensor_data): event start is at the given start time, and subsequent values start in 15 minute intervals throughout the one hour duration. The sensor is the one with ID=1. - The unit has to match the sensor's required unit. + The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but FlexMeasures will attempt to upsample lower resolutions. """ @@ -81,6 +81,8 @@ def get(self, sensor_data_description): "duration": "PT1H", "unit": "m³/h" } + + The unit has to be convertible from the sensor's unit. """ # todo: respect passed horizon and prior # todo: move some of the below logic to the dump_bdf (serialize) method on the schema From 1a7414ec8c635cd8b5706164651ea6b699db7e6b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 16:48:08 +0100 Subject: [PATCH 17/99] Respect the passed horizon and prior fields Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index a8826486a..026a26e11 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -84,7 +84,6 @@ def get(self, sensor_data_description): The unit has to be convertible from the sensor's unit. """ - # todo: respect passed horizon and prior # todo: move some of the below logic to the dump_bdf (serialize) method on the schema sensor: Sensor = sensor_data_description["sensor"] start = sensor_data_description["start"] @@ -96,6 +95,8 @@ def get(self, sensor_data_description): sensor.search_beliefs( event_starts_after=start, event_ends_before=end, + horizons_at_least=sensor_data_description.get("horizon", None), + beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, as_json=False, ) From 70e379315ae1dd74710fc0d34df6f65b85631b36 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 17:19:04 +0100 Subject: [PATCH 18/99] Convert NaN to null (otherwise invalid JSON) Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 026a26e11..a13651ad8 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -106,21 +106,24 @@ def get(self, sensor_data_description): index = pd.date_range( start=start, end=end, freq=sensor.event_resolution, closed="left" ) - df = df.reindex(index).reset_index() + df = df.reindex(index) # Convert to desired unit values = convert_units( df["event_value"], from_unit=sensor.unit, to_unit=unit, - ).to_list() + ) + + # Convert NaN to null + values = values.where(pd.notnull(values), None) + # Form the response response = dict( - values=values, + values=values.tolist(), start=datetime_isoformat(start), duration=duration_isoformat(duration), unit=unit, ) - response.update(type="GetSensorDataResponse") return json.dumps(response) From 2ef234fc6d43450963b17625f2eeab031fd377db Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 17:36:27 +0100 Subject: [PATCH 19/99] Pass event resolution to enable additional unit conversions Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index a13651ad8..d6dfc190b 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -113,6 +113,7 @@ def get(self, sensor_data_description): df["event_value"], from_unit=sensor.unit, to_unit=unit, + event_resolution=sensor.event_resolution, ) # Convert NaN to null From bc5ffbeafb9b770380943c0f34fd054765ae98d7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 17:40:30 +0100 Subject: [PATCH 20/99] Smarter unit conversion for BeliefsSeries Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 1 - flexmeasures/utils/unit_utils.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index d6dfc190b..a13651ad8 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -113,7 +113,6 @@ def get(self, sensor_data_description): df["event_value"], from_unit=sensor.unit, to_unit=unit, - event_resolution=sensor.event_resolution, ) # Convert NaN to null diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 96e69608c..a8343f7e5 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd import pint +import timely_beliefs as tb # Edit constants template to stop using h to represent planck_constant constants_template = ( @@ -174,7 +175,7 @@ def is_energy_unit(unit: str) -> bool: def convert_units( - data: Union[pd.Series, List[Union[int, float]], int, float], + data: Union[tb.BeliefsSeries, pd.Series, List[Union[int, float]], int, float], from_unit: str, to_unit: str, event_resolution: Optional[timedelta] = None, @@ -216,6 +217,8 @@ def convert_units( ) else: # Catch multiplicative conversions that use the resolution, like "kWh/15min" to "kW" + if event_resolution is None and hasattr(data, "event_resolution"): + event_resolution = data.event_resolution multiplier = determine_unit_conversion_multiplier( from_unit, to_unit, event_resolution ) From 3e85a5ec22e07deae6869e7f9468e63b4e93a9c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 17:59:44 +0100 Subject: [PATCH 21/99] Revert to auth_token_required instead of login_required Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index a13651ad8..7d5bf6152 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -3,7 +3,7 @@ from isodate import datetime_isoformat, duration_isoformat import pandas as pd from flask_classful import FlaskView, route -from flask_security import login_required +from flask_security import auth_token_required from webargs.flaskparser import use_args from flexmeasures.api.common.schemas.sensor_data import ( From 5738a29f390e4d1d84f8ce3b516b93cb3d669404 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 18:01:28 +0100 Subject: [PATCH 22/99] Deserialize to BeliefsDataFrame using post_load decorator, and run through post_load decorators in a specific sequence Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 16 ++++++++++++---- .../api/v2_0/implementations/sensor_data.py | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 2b6264f51..1121f1541 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -134,7 +134,14 @@ def check_resolution_compatibility_of_values(self, data, **kwargs): ) @post_load() - def possibly_convert_units(self, data, **kwargs): + def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: + """If needed, upsample and convert units, then deserialize to a BeliefsDataFrame.""" + data = self.possibly_upsample_values(data) + data = self.possibly_convert_units(data) + return self.load_bdf(data) + + @staticmethod + def possibly_convert_units(data): """ Convert values if needed, to fit the sensor's unit. Marshmallow runs this after validation. @@ -147,8 +154,8 @@ def possibly_convert_units(self, data, **kwargs): ) return data - @post_load() - def possibly_upsample_values(self, data, **kwargs): + @staticmethod + def possibly_upsample_values(data): """ Upsample the data if needed, to fit to the sensor's resolution. Marshmallow runs this after validation. @@ -169,7 +176,8 @@ def possibly_upsample_values(self, data, **kwargs): ) return data - def load_bdf(sensor_data) -> BeliefsDataFrame: + @staticmethod + def load_bdf(sensor_data: dict) -> BeliefsDataFrame: """ Turn the de-serialized and validated data into a BeliefsDataFrame. """ diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 7d5bf6152..2551114ad 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -4,6 +4,7 @@ import pandas as pd from flask_classful import FlaskView, route from flask_security import auth_token_required +from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args from flexmeasures.api.common.schemas.sensor_data import ( @@ -20,7 +21,7 @@ class SensorDataAPI(FlaskView): route_base = "/sensorData" - decorators = [login_required] + decorators = [auth_token_required] @account_roles_accepted("MDC", "Prosumer") @use_args( @@ -28,7 +29,7 @@ class SensorDataAPI(FlaskView): location="json", ) @route("/", methods=["POST"]) - def post(self, sensor_data): + def post(self, bdf: BeliefsDataFrame): """ Post sensor data to FlexMeasures. @@ -55,8 +56,7 @@ def post(self, sensor_data): The resolution of the data has to match the sensor's required resolution, but FlexMeasures will attempt to upsample lower resolutions. """ - beliefs = SensorDataSchema.load_bdf(sensor_data) - response, code = save_and_enqueue(beliefs) + response, code = save_and_enqueue(bdf) response.update(type="PostSensorDataResponse") return response, code From 9cd8f9b29fc0185ae2c49ccf903d9284f0aa7711 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 19:54:42 +0100 Subject: [PATCH 23/99] Move serialization logic to schema Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 58 +++++++++++++++++-- .../schemas/tests/test_sensor_data_schema.py | 4 +- .../api/v2_0/implementations/sensor_data.py | 56 ++---------------- 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 1121f1541..60dbbf44c 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -2,6 +2,7 @@ from typing import List, Union from flask_login import current_user +from isodate import datetime_isoformat, duration_isoformat from marshmallow import fields, post_load, validates_schema, ValidationError from marshmallow.validate import Equal, OneOf from marshmallow_polyfield import PolyField @@ -10,9 +11,11 @@ from flexmeasures.data import ma from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.time_series import Sensor from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField +from flexmeasures.data.services.time_series import simplify_index from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.unit_utils import ( convert_units, @@ -60,10 +63,7 @@ def select_schema_to_ensure_list_of_floats( class SensorDataDescriptionSchema(ma.Schema): """ - Describing sensor data (i.e. in a GET request). - - TODO: when we want to support other entity types with this - schema (assets/weather/markets or actuators), we'll need some re-design. + Describing sensor data request (i.e. in a GET request). """ type = fields.Str(required=True, validate=Equal("GetSensorDataRequest")) @@ -103,7 +103,55 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): ) -class SensorDataSchema(SensorDataDescriptionSchema): +class GetSensorDataSchema(SensorDataDescriptionSchema): + + @post_load + def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: + sensor: Sensor = sensor_data_description["sensor"] + start = sensor_data_description["start"] + duration = sensor_data_description["duration"] + end = sensor_data_description["start"] + duration + unit = sensor_data_description["unit"] + + df = simplify_index( + sensor.search_beliefs( + event_starts_after=start, + event_ends_before=end, + horizons_at_least=sensor_data_description.get("horizon", None), + beliefs_before=sensor_data_description.get("prior", None), + one_deterministic_belief_per_event=True, + as_json=False, + ) + ) + + # Convert to desired time range + index = pd.date_range( + start=start, end=end, freq=sensor.event_resolution, closed="left" + ) + df = df.reindex(index) + + # Convert to desired unit + values = convert_units( + df["event_value"], + from_unit=sensor.unit, + to_unit=unit, + ) + + # Convert NaN to null + values = values.where(pd.notnull(values), None) + + # Form the response + response = dict( + values=values.tolist(), + start=datetime_isoformat(start), + duration=duration_isoformat(duration), + unit=unit, + ) + + return response + + +class PostSensorDataSchema(SensorDataDescriptionSchema): """ This schema includes data, so it can be used for POST requests or GET responses. diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 2862fc558..07e859e10 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -4,7 +4,7 @@ from flexmeasures.api.common.schemas.sensor_data import ( SingleValueField, - SensorDataSchema, + PostSensorDataSchema, ) @@ -34,7 +34,7 @@ def test_value_field_deserialization( exp_deserialization_output, ): """Testing straightforward cases""" - vf = SensorDataSchema._declared_fields["values"] + vf = PostSensorDataSchema._declared_fields["values"] deser = vf.deserialize(deserialization_input) assert deser == exp_deserialization_output diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 2551114ad..11faf28d3 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -1,21 +1,16 @@ import json -from isodate import datetime_isoformat, duration_isoformat -import pandas as pd from flask_classful import FlaskView, route from flask_security import auth_token_required from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args from flexmeasures.api.common.schemas.sensor_data import ( - SensorDataSchema, - SensorDataDescriptionSchema, + GetSensorDataSchema, + PostSensorDataSchema, ) from flexmeasures.api.common.utils.api_utils import save_and_enqueue -from flexmeasures import Sensor from flexmeasures.auth.decorators import account_roles_accepted -from flexmeasures.data.services.time_series import simplify_index -from flexmeasures.utils.unit_utils import convert_units class SensorDataAPI(FlaskView): @@ -25,7 +20,7 @@ class SensorDataAPI(FlaskView): @account_roles_accepted("MDC", "Prosumer") @use_args( - SensorDataSchema(), + PostSensorDataSchema(), location="json", ) @route("/", methods=["POST"]) @@ -62,10 +57,10 @@ def post(self, bdf: BeliefsDataFrame): @route("/", methods=["GET"]) @use_args( - SensorDataDescriptionSchema(), + GetSensorDataSchema(), location="query", ) - def get(self, sensor_data_description): + def get(self, response: dict): """Get sensor data from FlexMeasures. .. :quickref: Data; Download sensor data @@ -84,46 +79,5 @@ def get(self, sensor_data_description): The unit has to be convertible from the sensor's unit. """ - # todo: move some of the below logic to the dump_bdf (serialize) method on the schema - sensor: Sensor = sensor_data_description["sensor"] - start = sensor_data_description["start"] - duration = sensor_data_description["duration"] - end = sensor_data_description["start"] + duration - unit = sensor_data_description["unit"] - - df = simplify_index( - sensor.search_beliefs( - event_starts_after=start, - event_ends_before=end, - horizons_at_least=sensor_data_description.get("horizon", None), - beliefs_before=sensor_data_description.get("prior", None), - one_deterministic_belief_per_event=True, - as_json=False, - ) - ) - - # Convert to desired time range - index = pd.date_range( - start=start, end=end, freq=sensor.event_resolution, closed="left" - ) - df = df.reindex(index) - - # Convert to desired unit - values = convert_units( - df["event_value"], - from_unit=sensor.unit, - to_unit=unit, - ) - - # Convert NaN to null - values = values.where(pd.notnull(values), None) - - # Form the response - response = dict( - values=values.tolist(), - start=datetime_isoformat(start), - duration=duration_isoformat(duration), - unit=unit, - ) response.update(type="GetSensorDataResponse") return json.dumps(response) From c76766ead16395a3cdc95ea377da4f966b3d8772 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 19:56:04 +0100 Subject: [PATCH 24/99] Fix tutorial for posting data Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 201fec5e4..9dcf68f10 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -31,13 +31,20 @@ Posting sensor data Sensor data (both observations and forecasts) can be posted to `POST /api/v2_0/postSensorData <../api/v2_0.html#post--api-v2_0-postSensorData>`_. This endpoint represents the basic method of getting time series data into FlexMeasures via API. -It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that happened in the past or in the future. -Some examples: readings from electricity meters, gas meters, temperature sensors and pressure sensors, the availability of parking spots, and price forecasts. +It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that have happened in the past or will happen in the future. +Some examples: + +- readings from electricity and gas meters +- readings from temperature and pressure sensors +- state of charge of a battery +- estimated availability of parking spots +- price forecasts + The exact URL will depend on your domain name, and will look approximately like this: .. code-block:: html - https://company.flexmeasures.io/api//postSensorData + [POST] https://company.flexmeasures.io/api//sensorData This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. From 271c43574b27458b12445913648877a464badbed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 21:09:46 +0100 Subject: [PATCH 25/99] Introduce util function to check for a valid price unit Signed-off-by: F.N. Claessen --- flexmeasures/utils/unit_utils.py | 44 ++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index a8343f7e5..99ed5bb0c 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -152,10 +152,14 @@ def units_are_convertible( def is_power_unit(unit: str) -> bool: """For example: - >>> is_power_unit("kW") # True - >>> is_power_unit("°C") # False - >>> is_power_unit("kWh") # False - >>> is_power_unit("EUR/MWh") # False + >>> is_power_unit("kW") + True + >>> is_power_unit("°C") + False + >>> is_power_unit("kWh") + False + >>> is_power_unit("EUR/MWh") + False """ if not is_valid_unit(unit): return False @@ -164,16 +168,40 @@ def is_power_unit(unit: str) -> bool: def is_energy_unit(unit: str) -> bool: """For example: - >>> is_energy_unit("kW") # False - >>> is_energy_unit("°C") # False - >>> is_energy_unit("kWh") # True - >>> is_energy_unit("EUR/MWh") # False + >>> is_energy_unit("kW") + False + >>> is_energy_unit("°C") + False + >>> is_energy_unit("kWh") + True + >>> is_energy_unit("EUR/MWh") + False """ if not is_valid_unit(unit): return False return ur.Quantity(unit).dimensionality == ur.Quantity("Wh").dimensionality +def is_energy_price_unit(unit: str) -> bool: + """For example: + >>> is_energy_price_unit("EUR/MWh") + True + >>> is_energy_price_unit("KRW/MWh") + True + >>> is_energy_price_unit("KRW/MW") + False + >>> is_energy_price_unit("beans/MW") + False + """ + if ( + unit[:3] in [str(c) for c in list_all_currencies()] + and unit[3] == "/" + and is_energy_unit(unit[4:]) + ): + return True + return False + + def convert_units( data: Union[tb.BeliefsSeries, pd.Series, List[Union[int, float]], int, float], from_unit: str, From 56a5f2527c9ad0f4cf873151ddd5e873186e50a0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 21:43:02 +0100 Subject: [PATCH 26/99] Comment out tutorial sections on posting data for multiple connections at once Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 139 +++++++++++++++-------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 9dcf68f10..75cc7879a 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -153,85 +153,88 @@ Multiple values (indicating a univariate timeseries) for 15-minute time interval "unit": "MW" } -Single identical value, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. + todo: uncomment whenever the new sensor data API supports sending data for multiple sensors in one message -Single identical value for a 15-minute time interval for two connections, posted 5 minutes after realisation. -Please note that both connections consumed at 10 MW, i.e. the value does not represent the total of the two connections. -We recommend to use this notation for zero values only. + Single identical value, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Single identical value for a 15-minute time interval for two sensors, posted 5 minutes after realisation. + Please note that both sensors consumed at 10 MW, i.e. the value does not represent the total of the two sensors. + We recommend to use this notation for zero values only. - { - "type": "PostMeterDataRequest", - "connections": [ - "ea1.2021-01.io.flexmeasures.company:fm1.1", - "ea1.2021-01.io.flexmeasures.company:fm1.2" - ], - "value": 10, - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json -Single different values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + { + "type": "PostSensorDataRequest", + "sensors": [ + "ea1.2021-01.io.flexmeasures.company:fm1.1", + "ea1.2021-01.io.flexmeasures.company:fm1.2" + ], + "value": 10, + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } -Single different values for a 15-minute time interval for two connections, posted 5 minutes after realisation. + Single different values, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Single different values for a 15-minute time interval for two sensors, posted 5 minutes after realisation. - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", - "value": 220 - }, - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.2", - "value": 300 - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json -Multiple values, multiple connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + { + "type": "PostSensorDataRequest", + "groups": [ + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "value": 220 + }, + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.2", + "value": 300 + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H15M", + "horizon": "-PT5M", + "unit": "MW" + } -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two connections, posted 5 minutes after each realisation. + Multiple values, multiple sensors + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json + Multiple values (indicating a univariate timeseries) for 15-minute time intervals for two sensors, posted 5 minutes after each realisation. - { - "type": "PostMeterDataRequest", - "groups": [ - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", - "values": [ - 220, - 210, - 200 - ] - }, - { - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.2", - "values": [ - 300, - 303, - 306 - ] - } - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H45M", - "horizon": "-PT5M", - "unit": "MW" - } + .. code-block:: json + + { + "type": "PostSensorDataRequest", + "groups": [ + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "values": [ + 220, + 210, + 200 + ] + }, + { + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.2", + "values": [ + 300, + 303, + 306 + ] + } + ], + "start": "2015-01-01T00:00:00+00:00", + "duration": "PT0H45M", + "horizon": "-PT5M", + "unit": "MW" + } .. _observations_vs_forecasts From 8a14c647c3182cf82c4a8d1c589f045c6d7b93d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 21:45:39 +0100 Subject: [PATCH 27/99] Update tutorial for posting data: change connection to sensor and stop mentioning separate endpoints for postMeterData and postPrognosis Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 48 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 75cc7879a..d8ea63ac6 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -26,6 +26,8 @@ Prerequisites .. note:: To address assets and sensors, these tutorials assume entity addresses valid in the namespace ``fm1``. See :ref:`api_introduction` for more explanations. +.. _posting_sensor_data: + Posting sensor data ------------------- @@ -54,7 +56,7 @@ The ``prior`` indicates that the prices were published at 3pm on December 31st 2 { "type": "PostSensorDataRequest", - "market": "ea1.2021-01.io.flexmeasures.company:fm1.16", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.16", "values": [ 52.37, 51.14, @@ -96,35 +98,33 @@ Posting power data ------------------ For power data, USEF specifies separate message types for observations and forecasts. -Correspondingly, FlexMeasures uses separate endpoints to communicate these messages. -Observations of power data can be posted to `POST /api/v2_0/postMeterData <../api/v2_0.html#post--api-v2_0-postMeterData>`_. The URL might look like this: - -.. code-block:: html - - https://company.flexmeasures.io/api//postMeterData +Correspondingly, we allow the following message types to be used with the [POST] /sensorData endpoint (see :ref:`posting_sensor_data`): -while forecasts of power data can be posted to `POST /api/v2_0/postPrognosis <../api/v2_0.html#post--api-v2_0-postPrognosis>`_. The URL might look like this: - -.. code-block:: html +.. code-block:: json - https://company.flexmeasures.io/api//postPrognosis + { + "type": "PostMeterDataRequest" + } -For both endpoints, power data can be posted in various ways. -The following examples assume that the endpoint for power data observations (i.e. meter data) is used. +.. code-block:: json -.. todo:: For the time being, only one rate unit (MW) can be used to post power values. + { + "type": "PostPrognosisRequest" + } +For these message types, FlexMeasures validates whether the data unit is suitable for communicating power data. +Additionally, we validate whether meter data lies in the past, and prognoses lie in the future. -Single value, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Single value, single sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A single average power value for a 15-minute time interval for a single connection, posted 5 minutes after realisation. +A single average power value for a 15-minute time interval for a single sensor, posted 5 minutes after realisation. .. code-block:: json { - "type": "PostMeterDataRequest", - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", "value": 220, "start": "2015-01-01T00:00:00+00:00", "duration": "PT0H15M", @@ -132,16 +132,16 @@ A single average power value for a 15-minute time interval for a single connecti "unit": "MW" } -Multiple values, single connection -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Multiple values, single sensor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single connection, posted 5 minutes after each realisation. +Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single sensor, posted 5 minutes after each realisation. .. code-block:: json { - "type": "PostMeterDataRequest", - "connection": "ea1.2021-01.io.flexmeasures.company:fm1.1", + "type": "PostSensorDataRequest", + "sensor": "ea1.2021-01.io.flexmeasures.company:fm1.1", "values": [ 220, 210, From e878f5c6f28f39663292319aae6baf1a35136c68 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 21:47:19 +0100 Subject: [PATCH 28/99] Add validation based on POST message type Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 60dbbf44c..59e63ccac 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -20,6 +20,7 @@ from flexmeasures.utils.unit_utils import ( convert_units, units_are_convertible, + is_energy_price_unit, ) @@ -161,7 +162,15 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): """ type = fields.Str( - validate=OneOf(["PostSensorDataRequest", "GetSensorDataResponse"]) + validate=OneOf( + [ + "PostSensorDataRequest", + "PostMeterDataRequest", + "PostPrognosisRequest", + "PostPriceDataRequest", + "PostWeatherDataRequest", + ] + ) ) values = PolyField( deserialization_schema_selector=select_schema_to_ensure_list_of_floats, @@ -169,6 +178,26 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): many=False, ) + @validates_schema + def check_schema_unit_against_type(self, data, **kwargs): + posted_unit = data["unit"] + _type = data["type"] + if ( + _type + in ( + "PostMeterDataRequest", + "PostPrognosisRequest", + ) + and not units_are_convertible(posted_unit, "MW") + ): + raise ValidationError( + f"The unit required for this message type should be convertible to MW, got incompatible unit: {posted_unit}" + ) + elif _type == "PostPriceDataRequest" and not is_energy_price_unit(posted_unit): + raise ValidationError( + f"The unit required for this message type should be convertible to an energy price unit, got incompatible unit: {posted_unit}" + ) + @validates_schema def check_resolution_compatibility_of_values(self, data, **kwargs): inferred_resolution = data["duration"] / len(data["values"]) @@ -186,7 +215,18 @@ def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: """If needed, upsample and convert units, then deserialize to a BeliefsDataFrame.""" data = self.possibly_upsample_values(data) data = self.possibly_convert_units(data) - return self.load_bdf(data) + bdf = self.load_bdf(data) + + # Post-load validation against message type + _type = data["type"] + if _type == "PostMeterDataRequest": + if any(h > timedelta(0) for h in bdf.belief_horizons): + raise ValidationError("Meter data must lie in the past.") + elif _type == "PostPrognosisRequest": + if any(h < timedelta(0) for h in bdf.belief_horizons): + raise ValidationError("Prognoses must lie in the future.") + + return bdf @staticmethod def possibly_convert_units(data): From dfa64b8d132c8bc284c7bc114b7cf941d815228c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 21:53:59 +0100 Subject: [PATCH 29/99] Publish documentation for SensorDataAPI Signed-off-by: F.N. Claessen --- documentation/api/v2_0.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst index 131995f91..2d85f9896 100644 --- a/documentation/api/v2_0.rst +++ b/documentation/api/v2_0.rst @@ -6,6 +6,11 @@ Version 2.0 Summary ------- +.. qrefflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v2_0.implementations.sensor_data + :order: path + :include-empty-docstring: + .. qrefflask:: flexmeasures.app:create(env="documentation") :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path @@ -14,6 +19,11 @@ Summary API Details ----------- +.. autoflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v2_0.implementations.sensor_data + :order: path + :include-empty-docstring: + .. autoflask:: flexmeasures.app:create(env="documentation") :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path From 7de8b6ff0ea45e1e5e2e896d7969559fafcd9b1e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 22:07:45 +0100 Subject: [PATCH 30/99] Add validation based on GET message type Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 59e63ccac..8540126c8 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -105,6 +105,38 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): + type = fields.Str( + validate=OneOf( + [ + "GetSensorDataRequest", + "GetMeterDataRequest", + "GetPrognosisRequest", + "GetPriceDataRequest", + ] + ) + ) + + @validates_schema + def check_schema_unit_against_type(self, data, **kwargs): + requested_unit = data["unit"] + _type = data["type"] + if ( + _type + in ( + "GetMeterDataRequest", + "GetPrognosisRequest", + ) + and not units_are_convertible(requested_unit, "MW") + ): + raise ValidationError( + f"The unit requested for this message type should be convertible from MW, got incompatible unit: {requested_unit}" + ) + elif _type == "GetPriceDataRequest" and not is_energy_price_unit( + requested_unit + ): + raise ValidationError( + f"The unit requested for this message type should be convertible from an energy price unit, got incompatible unit: {requested_unit}" + ) @post_load def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: @@ -114,11 +146,25 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: end = sensor_data_description["start"] + duration unit = sensor_data_description["unit"] + # Post-load configuration of belief timing against message type + horizons_at_least = sensor_data_description.get("horizon", None) + horizons_at_most = None + _type = sensor_data_description["type"] + if _type == "PostMeterDataRequest": + horizons_at_most = timedelta(0) + elif _type == "PostPrognosisRequest": + if horizons_at_least is None: + horizons_at_least = timedelta(0) + else: + # If the horizon field is used, ensure we still respect the minimum horizon for prognoses + horizons_at_least = max(horizons_at_least, timedelta(0)) + df = simplify_index( sensor.search_beliefs( event_starts_after=start, event_ends_before=end, - horizons_at_least=sensor_data_description.get("horizon", None), + horizons_at_least=horizons_at_least, + horizons_at_most=horizons_at_most, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, as_json=False, From c0f0bee9d29e3ef8c61f49d4206753f47e30aa89 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 22:20:21 +0100 Subject: [PATCH 31/99] mypy Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 2 +- flexmeasures/utils/unit_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 8540126c8..bd14166dc 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -178,7 +178,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: df = df.reindex(index) # Convert to desired unit - values = convert_units( + values: pd.Series = convert_units( # type: ignore df["event_value"], from_unit=sensor.unit, to_unit=unit, diff --git a/flexmeasures/utils/unit_utils.py b/flexmeasures/utils/unit_utils.py index 99ed5bb0c..cfeeaa791 100644 --- a/flexmeasures/utils/unit_utils.py +++ b/flexmeasures/utils/unit_utils.py @@ -245,7 +245,7 @@ def convert_units( ) else: # Catch multiplicative conversions that use the resolution, like "kWh/15min" to "kW" - if event_resolution is None and hasattr(data, "event_resolution"): + if event_resolution is None and isinstance(data, tb.BeliefsSeries): event_resolution = data.event_resolution multiplier = determine_unit_conversion_multiplier( from_unit, to_unit, event_resolution From 29d3381b91ab4183128a9692dfa3374a7c1bd500 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 22:26:53 +0100 Subject: [PATCH 32/99] Update getting-started.rst Signed-off-by: F.N. Claessen --- documentation/getting-started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 200046fe8..923b9b2f1 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -195,7 +195,7 @@ First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` This assumes you have a file `my-data.csv` with measurements, which was exported from some legacy database, and that the data is about our sensor with ID 1. This command has many options, so do use its ``--help`` function. -Second, you can use the `POST /api/v2_0/postMeterData `_ endpoint in the FlexMeasures API to send meter data. +Second, you can use the `POST /api/v2_0/postSensorData `_ endpoint in the FlexMeasures API to send meter data. Finally, you can tell FlexMeasures to create forecasts for your meter data with the ``flexmeasures add forecasts`` command, here is an example: From 852bee847d9bc7c160dd9cbdc9ca3c2e1e9804d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 8 Mar 2022 22:38:04 +0100 Subject: [PATCH 33/99] Undoc old endpoints Signed-off-by: F.N. Claessen --- documentation/api/v2_0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst index 2d85f9896..c79fe56e7 100644 --- a/documentation/api/v2_0.rst +++ b/documentation/api/v2_0.rst @@ -15,6 +15,7 @@ Summary :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path :include-empty-docstring: + :undoc-endpoints: flexmeasures_api_v2_0.get_meter_data, flexmeasures_api_v2_0.get_prognosis, flexmeasures_api_v2_0.post_meter_data, flexmeasures_api_v2_0.post_price_data, flexmeasures_api_v2_0.post_prognosis, flexmeasures_api_v2_0.post_weather_data API Details ----------- @@ -28,3 +29,4 @@ API Details :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path :include-empty-docstring: + :undoc-endpoints: flexmeasures_api_v2_0.get_meter_data, flexmeasures_api_v2_0.get_prognosis, flexmeasures_api_v2_0.post_meter_data, flexmeasures_api_v2_0.post_price_data, flexmeasures_api_v2_0.post_prognosis, flexmeasures_api_v2_0.post_weather_data From eef5d427e2999498cae6f89b3bebd96e6b7f3092 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 10 Mar 2022 14:08:45 +0100 Subject: [PATCH 34/99] Rename with_appcontext_if_needed Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 4 ++-- flexmeasures/data/schemas/sensors.py | 4 ++-- flexmeasures/data/schemas/utils.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index a8ef0058c..3fa6ef52a 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -8,7 +8,7 @@ from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, - with_appcontext, + with_appcontext_if_needed, ) @@ -99,7 +99,7 @@ class Meta: class GenericAssetIdField(fields.Int, MarshmallowClickMixin): """Field that deserializes to a GenericAsset and serializes back to an integer.""" - @with_appcontext() + @with_appcontext_if_needed() def _deserialize(self, value, attr, obj, **kwargs) -> GenericAsset: """Turn a generic asset id into a GenericAsset.""" generic_asset = GenericAsset.query.get(value) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 1eca8b4bd..62b14d853 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -6,7 +6,7 @@ from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, - with_appcontext, + with_appcontext_if_needed, ) from flexmeasures.utils.unit_utils import is_valid_unit @@ -58,7 +58,7 @@ class Meta: class SensorIdField(fields.Int, MarshmallowClickMixin): """Field that deserializes to a Sensor and serializes back to an integer.""" - @with_appcontext() + @with_appcontext_if_needed() def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: """Turn a sensor id into a Sensor.""" sensor = Sensor.query.get(value) diff --git a/flexmeasures/data/schemas/utils.py b/flexmeasures/data/schemas/utils.py index a7c208e3a..e5be8504f 100644 --- a/flexmeasures/data/schemas/utils.py +++ b/flexmeasures/data/schemas/utils.py @@ -28,7 +28,7 @@ class FMValidationError(ValidationError): status = "UNPROCESSABLE_ENTITY" -def with_appcontext(): +def with_appcontext_if_needed(): """Execute within the script's application context, in case there is one.""" def decorator(f): From 83b774ed8738e9c9f7cdcfaa1313aa5ec3e3f913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 10 Mar 2022 23:03:55 +0100 Subject: [PATCH 35/99] refactor auth logic - separate checking access from the permission_required_for_context decorator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/auth/decorators.py | 39 +++++-------------------------- flexmeasures/auth/policy.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 75af4c0ad..653c49fe7 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -8,15 +8,9 @@ roles_required as roles_required_fs, ) from werkzeug.local import LocalProxy -from werkzeug.exceptions import Forbidden, Unauthorized - -from flexmeasures.auth.policy import ( - ADMIN_ROLE, - PERMISSIONS, - AuthModelMixin, - user_has_admin_access, - user_matches_principals, -) +from werkzeug.exceptions import Forbidden + +from flexmeasures.auth.policy import ADMIN_ROLE, AuthModelMixin, check_access _security = LocalProxy(lambda: current_app.extensions["security"]) @@ -143,10 +137,6 @@ def view(resource_id: int, the_resource: Resource): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): - if permission not in PERMISSIONS: - raise Forbidden(f"Permission '{permission}' cannot be handled.") - if current_user.is_anonymous: - raise Unauthorized() # load & check context if arg_loader is not None: context: AuthModelMixin = arg_loader() @@ -158,26 +148,9 @@ def decorated_view(*args, **kwargs): context = kwargs[arg_name] else: context = args[0] - if context is None: - raise Forbidden( - f"Context needs {permission}-permission, but no context was passed." - ) - if not isinstance(context, AuthModelMixin): - raise Forbidden( - f"Context {context} needs {permission}-permission, but is no AuthModelMixin." - ) - # now check access, either with admin rights or principal(s) - acl = context.__acl__() - principals = acl.get(permission, tuple()) - current_app.logger.debug( - f"Looking for {permission}-permission on {context} ... Principals: {principals}" - ) - if not user_has_admin_access( - current_user, permission - ) and not user_matches_principals(current_user, principals): - raise Forbidden( - f"Authorization failure (accessing {context} to {permission}) ― cannot match {current_user} against {principals}!" - ) + + check_access(context, permission) + return fn(*args, **kwargs) return decorated_view diff --git a/flexmeasures/auth/policy.py b/flexmeasures/auth/policy.py index 37e030fbb..c7a0e6b0d 100644 --- a/flexmeasures/auth/policy.py +++ b/flexmeasures/auth/policy.py @@ -1,6 +1,8 @@ from typing import Dict, Union, Tuple, List from flask import current_app +from flask_security import current_user +from werkzeug.exceptions import Unauthorized, Forbidden PERMISSIONS = ["create-children", "read", "read-children", "update", "delete"] @@ -69,6 +71,45 @@ def __acl__(self) -> Dict[str, PRINCIPALS_TYPE]: return {} +def check_access(context: AuthModelMixin, permission: str): + """ + Check if current user can access this auth context if this permission + is required, either with admin rights or principal(s). + + Raises 401 or 403 otherwise. + """ + # check current user + if permission not in PERMISSIONS: + raise Forbidden(f"Permission '{permission}' cannot be handled.") + if current_user.is_anonymous: + raise Unauthorized() + + # check context + if context is None: + raise Forbidden( + f"Context needs {permission}-permission, but no context was passed." + ) + if not isinstance(context, AuthModelMixin): + raise Forbidden( + f"Context {context} needs {permission}-permission, but is no AuthModelMixin." + ) + + # look up principals + acl = context.__acl__() + principals: PRINCIPALS_TYPE = acl.get(permission, []) + current_app.logger.debug( + f"Looking for {permission}-permission on {context} ... Principals: {principals}" + ) + + # check access + if not user_has_admin_access( + current_user, permission + ) and not user_matches_principals(current_user, principals): + raise Forbidden( + f"Authorization failure (accessing {context} to {permission}) ― cannot match {current_user} against {principals}!" + ) + + def user_has_admin_access(user, permission: str) -> bool: if user.has_role(ADMIN_ROLE) or ( user.has_role(ADMIN_READER_ROLE) and permission == "read" From 52b3ba95fc08672be77806309fe91a5b1d9abe4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 12 Mar 2022 22:31:14 +0100 Subject: [PATCH 36/99] add modern auth for sensor data get & post, remove checking account role auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/common/schemas/sensor_data.py | 18 +++++++++--------- .../api/v2_0/implementations/sensor_data.py | 2 -- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index bd14166dc..d0d55a671 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -22,6 +22,7 @@ units_are_convertible, is_energy_price_unit, ) +from flexmeasures.auth.policy import check_access class SingleValueField(fields.Float): @@ -75,15 +76,6 @@ class SensorDataDescriptionSchema(ma.Schema): prior = AwareDateTimeField(required=False, format="iso") unit = fields.Str(required=True) - @validates_schema - def check_user_rights_against_sensor(self, data, **kwargs): - """If the user is a Prosumer and the sensor belongs to an asset - over which the Prosumer has no ownership, raise a ValidationError. - """ - # todo: implement check once sensors can belong to an asset - # https://github.com/FlexMeasures/flexmeasures/issues/155 - pass - @validates_schema def check_schema_unit_against_sensor_unit(self, data, **kwargs): """Allows units compatible with that of the sensor. @@ -116,6 +108,10 @@ class GetSensorDataSchema(SensorDataDescriptionSchema): ) ) + @validates_schema + def check_user_may_read(self, data, **kwargs): + check_access(data["sensor"], "read") + @validates_schema def check_schema_unit_against_type(self, data, **kwargs): requested_unit = data["unit"] @@ -224,6 +220,10 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): many=False, ) + @validates_schema + def check_user_may_create(self, data, **kwargs): + check_access(data["sensor"], "create-children") + @validates_schema def check_schema_unit_against_type(self, data, **kwargs): posted_unit = data["unit"] diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 11faf28d3..4be34204a 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -10,7 +10,6 @@ PostSensorDataSchema, ) from flexmeasures.api.common.utils.api_utils import save_and_enqueue -from flexmeasures.auth.decorators import account_roles_accepted class SensorDataAPI(FlaskView): @@ -18,7 +17,6 @@ class SensorDataAPI(FlaskView): route_base = "/sensorData" decorators = [auth_token_required] - @account_roles_accepted("MDC", "Prosumer") @use_args( PostSensorDataSchema(), location="json", From 9c0695ed95426f3ad3b7ba05963af7b56e16f9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 12 Mar 2022 22:33:05 +0100 Subject: [PATCH 37/99] more freedom to create inside accounts: allow all account members to create on generic assets and sensors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/models/generic_assets.py | 6 +++--- flexmeasures/data/models/time_series.py | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index a3617477e..4f7ae95b1 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -68,11 +68,11 @@ class GenericAsset(db.Model, AuthModelMixin): def __acl__(self): """ All logged-in users can read if the asset is public. - Within same account, everyone can read and update. - Creation and deletion are left to account admins. + Within same account, everyone can create, read and update. + Deletion is left to account admins. """ return { - "create-children": (f"account:{self.account_id}", "role:account-admin"), + "create-children": f"account:{self.account_id}", "read": f"account:{self.account_id}" if self.account_id is not None else EVERY_LOGGED_IN_USER, diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index a509b840a..4faa6c35e 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -92,14 +92,11 @@ def __init__( def __acl__(self): """ All logged-in users can read if the sensor belongs to a public asset. - Within same account, everyone can read and update. - Creation and deletion are left to account admins. + Within same account, everyone can create, read and update. + Deletion is left to account admins. """ return { - "create-children": ( - f"account:{self.generic_asset.account_id}", - "role:account-admin", - ), + "create-children": f"account:{self.generic_asset.account_id}", "read": f"account:{self.generic_asset.account_id}" if self.generic_asset.account_id is not None else EVERY_LOGGED_IN_USER, From bc9a100139a86d7ef7b43360a711e6dcdcaa48bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 12 Mar 2022 23:35:45 +0100 Subject: [PATCH 38/99] use a user from the same account as the sensor for sensor data tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/dev/tests/conftest.py | 10 +++++----- flexmeasures/api/dev/tests/test_sensor_data.py | 9 ++++++--- .../api/dev/tests/test_sensor_data_fresh_db.py | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 623bf574c..df97ec761 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -12,7 +12,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) - add_gas_sensor(db, setup_roles_users["Test Prosumer User 2"]) + add_gas_sensor(db, setup_roles_users["Test Supplier User"]) @pytest.fixture(scope="function") @@ -25,10 +25,10 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Prosumer User 2"]) + add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) -def add_gas_sensor(db, test_supplier): +def add_gas_sensor(db, test_supplier_user): incineration_type = GenericAssetType( name="waste incinerator", ) @@ -37,7 +37,7 @@ def add_gas_sensor(db, test_supplier): incineration_asset = GenericAsset( name="incineration line", generic_asset_type=incineration_type, - account_id=test_supplier.account_id, + account_id=test_supplier_user.account_id, ) db.session.add(incineration_asset) db.session.flush() @@ -48,4 +48,4 @@ def add_gas_sensor(db, test_supplier): generic_asset=incineration_asset, ) db.session.add(gas_sensor) - gas_sensor.owner = test_supplier + gas_sensor.owner = test_supplier_user.account diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/dev/tests/test_sensor_data.py index ff660d2fc..2f1eb1124 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data.py +++ b/flexmeasures/api/dev/tests/test_sensor_data.py @@ -14,14 +14,16 @@ def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): headers = {"content-type": "application/json"} if use_auth: # in this case, we successfully authenticate, - # but fail authorization (no admin or MDC role) + # but fail authorization (not member of the account in which the sensor lies) headers["Authorization"] = get_auth_token( client, "test_dummy_user_3@seita.nl", "testtest" ) + post_data = make_sensor_data_request_for_gas_sensor() post_data_response = client.post( url_for("SensorDataAPI:post"), headers=headers, + json=post_data, ) print("Server responded with:\n%s" % post_data_response.data) if use_auth: @@ -51,7 +53,7 @@ def test_post_invalid_sensor_data( post_data = make_sensor_data_request_for_gas_sensor() post_data[request_field] = new_value # this guy is allowed to post sensorData - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( url_for("SensorDataAPI:post"), json=post_data, @@ -63,7 +65,7 @@ def test_post_invalid_sensor_data( def test_post_sensor_data_twice(client, setup_api_test_data): - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") post_data = make_sensor_data_request_for_gas_sensor() # Check that 1st time posting the data succeeds @@ -72,6 +74,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): json=post_data, headers={"Authorization": auth_token}, ) + print(response.json) assert response.status_code == 200 # Check that 2nd time posting the same data succeeds informatively diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py index 5ec661187..f8f878a63 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py @@ -37,7 +37,7 @@ def test_post_sensor_data( print(f"BELIEFS BEFORE: {beliefs_before}") assert len(beliefs_before) == 0 - auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( url_for("SensorDataAPI:post"), json=post_data, From 2f31a177a521adc7182cdc857f3e2a2850b317ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sat, 12 Mar 2022 23:52:54 +0100 Subject: [PATCH 39/99] adapt asset tests w.r.t. to a change in dev api conftest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/dev/tests/test_assets_api.py | 2 +- flexmeasures/api/dev/tests/test_assets_api_fresh_db.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/dev/tests/test_assets_api.py b/flexmeasures/api/dev/tests/test_assets_api.py index d28422569..646009bfe 100644 --- a/flexmeasures/api/dev/tests/test_assets_api.py +++ b/flexmeasures/api/dev/tests/test_assets_api.py @@ -70,7 +70,7 @@ def test_get_asset_nonaccount_access(client, setup_api_test_data): assert "not found" in asset_response.json["message"] -@pytest.mark.parametrize("account_name, num_assets", [("Prosumer", 2), ("Supplier", 1)]) +@pytest.mark.parametrize("account_name, num_assets", [("Prosumer", 1), ("Supplier", 2)]) def test_get_assets( client, setup_api_test_data, setup_accounts, account_name, num_assets ): diff --git a/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py b/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py index 119cb5ad0..676f9dd0f 100644 --- a/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_assets_api_fresh_db.py @@ -34,8 +34,8 @@ def test_post_an_asset_as_admin(client, setup_api_fresh_test_data, admin_kind): def test_edit_an_asset(client, setup_api_fresh_test_data): - with AccountContext("Test Prosumer Account") as prosumer: - existing_asset = prosumer.generic_assets[1] + with AccountContext("Test Supplier Account") as supplier: + existing_asset = supplier.generic_assets[1] post_data = dict(latitude=10, id=999) # id will be ignored auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") From 352ae6055d01458407e0c1fae21eeaaa0d40bd53 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 11 Mar 2022 16:55:29 +0100 Subject: [PATCH 40/99] Remove unneeded import Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/__init__.py b/flexmeasures/api/v2_0/implementations/__init__.py index e35323de0..c27fc08eb 100644 --- a/flexmeasures/api/v2_0/implementations/__init__.py +++ b/flexmeasures/api/v2_0/implementations/__init__.py @@ -1 +1 @@ -from . import assets, sensor_data, sensors, users # noqa F401 +from . import assets, sensors, users # noqa F401 From 210ecbc7618997491e75f758ab97cf9d7f7f33de Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 11 Mar 2022 16:57:35 +0100 Subject: [PATCH 41/99] GET endpoints work for logged-in users when tried in the browser with auth_required but not with auth_token_required Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index 4be34204a..dc43719a7 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -1,7 +1,7 @@ import json from flask_classful import FlaskView, route -from flask_security import auth_token_required +from flask_security import auth_required from timely_beliefs import BeliefsDataFrame from webargs.flaskparser import use_args @@ -15,7 +15,7 @@ class SensorDataAPI(FlaskView): route_base = "/sensorData" - decorators = [auth_token_required] + decorators = [auth_required()] @use_args( PostSensorDataSchema(), From b76ebdb31d37820353b0a0c69d5a0d76529a6502 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 11 Mar 2022 17:02:23 +0100 Subject: [PATCH 42/99] Add docstring Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index d0d55a671..7121663db 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -136,6 +136,14 @@ def check_schema_unit_against_type(self, data, **kwargs): @post_load def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: + """Turn the de-serialized and validated data description into a response. + + Specifically, this function: + - queries data according to the given description + - converts to a single deterministic belief per event + - ensures the response respects the requested time frame + - converts values to the requested unit + """ sensor: Sensor = sensor_data_description["sensor"] start = sensor_data_description["start"] duration = sensor_data_description["duration"] From 9b3f281910e69bb657c578b91a8ac3dd86ae0cdc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 11 Mar 2022 17:02:47 +0100 Subject: [PATCH 43/99] Make type a required field Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7121663db..66a1501e4 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -98,6 +98,7 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): type = fields.Str( + required=True, validate=OneOf( [ "GetSensorDataRequest", From 7b67a1fa247a5e99cb6040c70fa88af35e6a4bb4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 09:53:31 +0100 Subject: [PATCH 44/99] Move @route to top Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/implementations/sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v2_0/implementations/sensor_data.py index dc43719a7..e1fa5023a 100644 --- a/flexmeasures/api/v2_0/implementations/sensor_data.py +++ b/flexmeasures/api/v2_0/implementations/sensor_data.py @@ -17,11 +17,11 @@ class SensorDataAPI(FlaskView): route_base = "/sensorData" decorators = [auth_required()] + @route("/", methods=["POST"]) @use_args( PostSensorDataSchema(), location="json", ) - @route("/", methods=["POST"]) def post(self, bdf: BeliefsDataFrame): """ Post sensor data to FlexMeasures. From 2037394a5b09049e96e7fb2cf1b47de77fae6b03 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 09:57:56 +0100 Subject: [PATCH 45/99] SensorDataDescriptionSchema doesn't need the `type` field Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 66a1501e4..7641a9edb 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -65,10 +65,9 @@ def select_schema_to_ensure_list_of_floats( class SensorDataDescriptionSchema(ma.Schema): """ - Describing sensor data request (i.e. in a GET request). + Schema describing sensor data (specifically, the sensor and the timing of the data). """ - type = fields.Str(required=True, validate=Equal("GetSensorDataRequest")) sensor = SensorField(required=True, entity_type="sensor", fm_scheme="fm1") start = AwareDateTimeField(required=True, format="iso") duration = DurationField(required=True) From b17a9b8ce2c0029869d56e8c1c1b05317640823b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 10:45:16 +0100 Subject: [PATCH 46/99] Revert "Undoc old endpoints" This reverts commit 852bee847d9bc7c160dd9cbdc9ca3c2e1e9804d2. --- documentation/api/v2_0.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst index c79fe56e7..2d85f9896 100644 --- a/documentation/api/v2_0.rst +++ b/documentation/api/v2_0.rst @@ -15,7 +15,6 @@ Summary :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path :include-empty-docstring: - :undoc-endpoints: flexmeasures_api_v2_0.get_meter_data, flexmeasures_api_v2_0.get_prognosis, flexmeasures_api_v2_0.post_meter_data, flexmeasures_api_v2_0.post_price_data, flexmeasures_api_v2_0.post_prognosis, flexmeasures_api_v2_0.post_weather_data API Details ----------- @@ -29,4 +28,3 @@ API Details :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path :include-empty-docstring: - :undoc-endpoints: flexmeasures_api_v2_0.get_meter_data, flexmeasures_api_v2_0.get_prognosis, flexmeasures_api_v2_0.post_meter_data, flexmeasures_api_v2_0.post_price_data, flexmeasures_api_v2_0.post_prognosis, flexmeasures_api_v2_0.post_weather_data From 4e804ba6a14aae060ec230d72a183baa898792eb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 10:47:56 +0100 Subject: [PATCH 47/99] Move sensor data API documentation to v3 Signed-off-by: F.N. Claessen --- documentation/api/v2_0.rst | 10 ---------- documentation/api/v3_0.rst | 20 ++++++++++++++++++++ documentation/index.rst | 1 + 3 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 documentation/api/v3_0.rst diff --git a/documentation/api/v2_0.rst b/documentation/api/v2_0.rst index 2d85f9896..131995f91 100644 --- a/documentation/api/v2_0.rst +++ b/documentation/api/v2_0.rst @@ -6,11 +6,6 @@ Version 2.0 Summary ------- -.. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v2_0.implementations.sensor_data - :order: path - :include-empty-docstring: - .. qrefflask:: flexmeasures.app:create(env="documentation") :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path @@ -19,11 +14,6 @@ Summary API Details ----------- -.. autoflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v2_0.implementations.sensor_data - :order: path - :include-empty-docstring: - .. autoflask:: flexmeasures.app:create(env="documentation") :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v2_0 :order: path diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst new file mode 100644 index 000000000..6d53ff24a --- /dev/null +++ b/documentation/api/v3_0.rst @@ -0,0 +1,20 @@ +.. _v3_0: + +Version 3.0 +=========== + +Summary +------- + +.. qrefflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v3_0.implementations.sensor_data + :order: path + :include-empty-docstring: + +API Details +----------- + +.. autoflask:: flexmeasures.app:create(env="documentation") + :modules: flexmeasures.api.v3_0.implementations.sensor_data + :order: path + :include-empty-docstring: diff --git a/documentation/index.rst b/documentation/index.rst index 86aa47fec..9444fa478 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -128,6 +128,7 @@ The platform operator of FlexMeasures can be an Aggregator. :maxdepth: 1 api/introduction + api/v3_0 api/v2_0 api/v1_3 api/v1_2 From 24ffc1ad97f6b8d43c4850357623638b01609267 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 10:56:11 +0100 Subject: [PATCH 48/99] Move sensor data API to v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/__init__.py | 2 ++ flexmeasures/api/v2_0/__init__.py | 4 ---- flexmeasures/api/v3_0/__init__.py | 11 +++++++++++ flexmeasures/api/v3_0/implementations/__init__.py | 0 .../api/{v2_0 => v3_0}/implementations/sensor_data.py | 0 5 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 flexmeasures/api/v3_0/__init__.py create mode 100644 flexmeasures/api/v3_0/implementations/__init__.py rename flexmeasures/api/{v2_0 => v3_0}/implementations/sensor_data.py (100%) diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 3b104731b..9833ffd5e 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -110,6 +110,7 @@ def register_at(app: Flask): from flexmeasures.api.v1_2 import register_at as v1_2_register_at from flexmeasures.api.v1_3 import register_at as v1_3_register_at from flexmeasures.api.v2_0 import register_at as v2_0_register_at + from flexmeasures.api.v3_0 import register_at as v3_0_register_at from flexmeasures.api.dev import register_at as dev_register_at v1_register_at(app) @@ -117,4 +118,5 @@ def register_at(app: Flask): v1_2_register_at(app) v1_3_register_at(app) v2_0_register_at(app) + v3_0_register_at(app) dev_register_at(app) diff --git a/flexmeasures/api/v2_0/__init__.py b/flexmeasures/api/v2_0/__init__.py index 00b6597c3..b8c24c896 100644 --- a/flexmeasures/api/v2_0/__init__.py +++ b/flexmeasures/api/v2_0/__init__.py @@ -1,7 +1,5 @@ from flask import Flask, Blueprint -from flexmeasures.api.v2_0.implementations.sensor_data import SensorDataAPI - flexmeasures_api = Blueprint("flexmeasures_api_v2_0", __name__) @@ -13,5 +11,3 @@ def register_at(app: Flask): v2_0_api_prefix = "/api/v2_0" app.register_blueprint(flexmeasures_api, url_prefix=v2_0_api_prefix) - - SensorDataAPI.register(app, route_prefix=v2_0_api_prefix) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py new file mode 100644 index 000000000..61f6a11f9 --- /dev/null +++ b/flexmeasures/api/v3_0/__init__.py @@ -0,0 +1,11 @@ +from flask import Flask, Blueprint + +from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI + + +def register_at(app: Flask): + """This can be used to register this blueprint together with other api-related things""" + + v3_0_api_prefix = "/api/v3_0" + + SensorDataAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/implementations/__init__.py b/flexmeasures/api/v3_0/implementations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/v2_0/implementations/sensor_data.py b/flexmeasures/api/v3_0/implementations/sensor_data.py similarity index 100% rename from flexmeasures/api/v2_0/implementations/sensor_data.py rename to flexmeasures/api/v3_0/implementations/sensor_data.py From 77ffdaf6fdc019cf9872313ef9f414a7acdc44a7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 11:01:10 +0100 Subject: [PATCH 49/99] Move user API to class for v3 Signed-off-by: F.N. Claessen --- documentation/api/v3_0.rst | 4 +- flexmeasures/api/v3_0/__init__.py | 2 + .../api/v3_0/implementations/users.py | 241 ++++++++++++++++++ 3 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 flexmeasures/api/v3_0/implementations/users.py diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst index 6d53ff24a..c441b01d9 100644 --- a/documentation/api/v3_0.rst +++ b/documentation/api/v3_0.rst @@ -7,7 +7,7 @@ Summary ------- .. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.implementations.sensor_data + :modules: flexmeasures.api.v3_0.implementations.sensor_data, flexmeasures.api.v3_0.implementations.users :order: path :include-empty-docstring: @@ -15,6 +15,6 @@ API Details ----------- .. autoflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.implementations.sensor_data + :modules: flexmeasures.api.v3_0.implementations.sensor_data, flexmeasures.api.v3_0.implementations.users :order: path :include-empty-docstring: diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index 61f6a11f9..c909058fc 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,6 +1,7 @@ from flask import Flask, Blueprint from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI +from flexmeasures.api.v3_0.implementations.users import UserAPI def register_at(app: Flask): @@ -9,3 +10,4 @@ def register_at(app: Flask): v3_0_api_prefix = "/api/v3_0" SensorDataAPI.register(app, route_prefix=v3_0_api_prefix) + UserAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/implementations/users.py b/flexmeasures/api/v3_0/implementations/users.py new file mode 100644 index 000000000..e0e8e289d --- /dev/null +++ b/flexmeasures/api/v3_0/implementations/users.py @@ -0,0 +1,241 @@ +from flask_classful import FlaskView, route +from marshmallow import fields +from sqlalchemy.exc import IntegrityError +from webargs.flaskparser import use_kwargs, use_args +from flask_security import current_user +from flask_security.recoverable import send_reset_password_instructions +from flask_json import as_json +from werkzeug.exceptions import Forbidden + +from flexmeasures.data.models.user import User as UserModel, Account +from flexmeasures.api.common.schemas.users import AccountIdField, UserIdField +from flexmeasures.data.schemas.users import UserSchema +from flexmeasures.data.services.users import ( + get_users, + set_random_password, + remove_cookie_and_token_access, +) +from flexmeasures.auth.decorators import permission_required_for_context +from flexmeasures.data import db + +""" +API endpoints to manage users. + +Both POST (to create) and DELETE are not accessible via the API, but as CLI functions. +""" + +user_schema = UserSchema() +users_schema = UserSchema(many=True) + + +@use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + "include_inactive": fields.Bool(load_default=False), + }, + location="query", +) +@permission_required_for_context("read", arg_name="account") +@as_json +def get(account: Account, include_inactive: bool = False): + """List users of an account.""" + users = get_users(account_name=account.name, only_active=not include_inactive) + return users_schema.dump(users), 200 + + +@use_kwargs({"user": UserIdField(data_key="id")}, location="path") +@permission_required_for_context("read", arg_name="user") +@as_json +def fetch_one(user_id: int, user: UserModel): + """Fetch a given user""" + return user_schema.dump(user), 200 + + +@use_args(UserSchema(partial=True)) +@use_kwargs({"db_user": UserIdField(data_key="id")}, location="path") +@permission_required_for_context("update", arg_name="db_user") +@as_json +def patch(id: int, user_data: dict, db_user: UserModel): + """Update a user given its identifier""" + allowed_fields = ["email", "username", "active", "timezone", "flexmeasures_roles"] + for k, v in [(k, v) for k, v in user_data.items() if k in allowed_fields]: + if current_user.id == db_user.id and k in ("active", "flexmeasures_roles"): + raise Forbidden( + "Users who edit themselves cannot edit security-sensitive fields." + ) + setattr(db_user, k, v) + if k == "active" and v is False: + remove_cookie_and_token_access(db_user) + db.session.add(db_user) + try: + db.session.commit() + except IntegrityError as ie: + return dict(message="Duplicate user already exists", detail=ie._message()), 400 + return user_schema.dump(db_user), 200 + + +@use_kwargs({"user": UserIdField(data_key="id")}, location="path") +@permission_required_for_context("update", arg_name="user") +@as_json +def reset_password(user_id: int, user: UserModel): + """ + Reset the user's current password, cookies and auth tokens. + Send a password reset link to the user. + """ + set_random_password(user) + remove_cookie_and_token_access(user) + send_reset_password_instructions(user) + + # commit only if sending instructions worked, as well + db.session.commit() + + +class UserAPI(FlaskView): + route_base = "/user" + + @route("", methods=["GET"]) + def get_users(self): + """API endpoint to get users. + + .. :quickref: User; Download user list + + This endpoint returns all accessible users. + By default, only active users are returned. + The `include_inactive` query parameter can be used to also fetch + inactive users. + Accessible users are users in the same account as the current user. + Only admins can use this endpoint to fetch users from a different account (by using the `account_id` query parameter). + + **Example response** + + An example of one user being returned: + + .. sourcecode:: json + + [ + { + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'account_id': 13, + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + ] + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + """ + return get() + + @route("/", methods=["GET"]) + def get_user(self, id: int): + """API endpoint to get a user. + + .. :quickref: User; Get a user + + This endpoint gets a user. + Only admins or the user themselves can use this endpoint. + + **Example response** + + .. sourcecode:: json + + { + 'account_id': 1, + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + """ + return fetch_one(id) + + @route("/", methods=["PATCH"]) + def patch_user(self, id: int): + """API endpoint to patch user data. + + .. :quickref: User; Patch data for an existing user + + This endpoint sets data for an existing user. + Any subset of user fields can be sent. + Only the user themselves or admins are allowed to update its data, + while a non-admin can only edit a few of their own fields. + + Several fields are not allowed to be updated, e.g. id and account_id. They are ignored. + + **Example request** + + .. sourcecode:: json + + { + "active": false, + } + + **Example response** + + The whole user is returned in the response: + + .. sourcecode:: json + + { + 'account_id': 1, + 'active': True, + 'email': 'test_prosumer@seita.nl', + 'flexmeasures_roles': [1, 3], + 'id': 1, + 'timezone': 'Europe/Amsterdam', + 'username': 'Test Prosumer User' + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: UPDATED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + return patch(id) + + @route("//password-reset", methods=["PATCH"]) + def reset_user_password(self, id: int): + """API endpoint to reset the user password. They'll get an email to choose a new password. + + .. :quickref: User; Password reset + + Reset the user's password, and send them instructions on how to reset the password. + This endpoint is useful from a security standpoint, in case of worries the password might be compromised. + It sets the current password to something random, invalidates cookies and auth tokens, + and also sends an email for resetting the password to the user. + + Users can reset their own passwords. Only admins can use this endpoint to reset passwords of other users. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + """ + return reset_password(id) From daa43c64703f581c6260fd0643474bfe9493001f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 11:04:14 +0100 Subject: [PATCH 50/99] Move user UI to v3 Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/users.py | 10 ++++------ flexmeasures/ui/tests/test_user_crud.py | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/flexmeasures/ui/crud/users.py b/flexmeasures/ui/crud/users.py index 5e181e216..c7d4a7501 100644 --- a/flexmeasures/ui/crud/users.py +++ b/flexmeasures/ui/crud/users.py @@ -84,7 +84,7 @@ def index(self): for account in Account.query.all(): get_users_response = InternalApi().get( url_for( - "flexmeasures_api_v2_0.get_users", + "UserAPI:get_users", account_id=account.id, include_inactive=include_inactive, ) @@ -100,9 +100,7 @@ def index(self): @roles_required(ADMIN_ROLE) def get(self, id: str): """GET from /users/""" - get_user_response = InternalApi().get( - url_for("flexmeasures_api_v2_0.get_user", id=id) - ) + get_user_response = InternalApi().get(url_for("UserAPI:get_user", id=id)) user: User = process_internal_api_response( get_user_response.json(), make_obj=True ) @@ -119,7 +117,7 @@ def toggle_active(self, id: str): """Toggle activation status via /users/toggle_active/""" user: User = get_user(id) user_response = InternalApi().patch( - url_for("flexmeasures_api_v2_0.patch_user", id=id), + url_for("UserAPI:patch_user", id=id), args={"active": not user.active}, ) patched_user: User = process_internal_api_response( @@ -138,7 +136,7 @@ def reset_password_for(self, id: str): and send instructions on how to reset.""" user: User = get_user(id) InternalApi().patch( - url_for("flexmeasures_api_v2_0.reset_user_password", id=id), + url_for("UserAPI:reset_user_password", id=id), ) return render_user( user, diff --git a/flexmeasures/ui/tests/test_user_crud.py b/flexmeasures/ui/tests/test_user_crud.py index 71ed4543f..a2a698449 100644 --- a/flexmeasures/ui/tests/test_user_crud.py +++ b/flexmeasures/ui/tests/test_user_crud.py @@ -23,7 +23,7 @@ def test_user_crud_as_non_admin(client, as_prosumer_user1, view): def test_user_list(client, as_admin, requests_mock): requests_mock.get( - "http://localhost//api/v2_0/users", + "http://localhost//api/v3_0/user", status_code=200, json=mock_user_response(multiple=True), ) @@ -37,7 +37,7 @@ def test_user_list(client, as_admin, requests_mock): def test_user_page(client, as_admin, requests_mock): mock_user = mock_user_response(as_list=False) requests_mock.get( - "http://localhost//api/v2_0/user/2", status_code=200, json=mock_user + "http://localhost//api/v3_0/user/2", status_code=200, json=mock_user ) requests_mock.get( "http://localhost//api/v2_0/assets", @@ -55,7 +55,7 @@ def test_deactivate_user(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v2_0/user/{user2.id}", + f"http://localhost//api/v3_0/user/{user2.id}", status_code=200, json={"active": False}, ) @@ -72,7 +72,7 @@ def test_reset_password(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v2_0/user/{user2.id}/password-reset", + f"http://localhost//api/v3_0/user/{user2.id}/password-reset", status_code=200, ) user_page = client.get( From 3abb6491199a85165e921092ef869ce0c436a802 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 11:04:43 +0100 Subject: [PATCH 51/99] Update user API tests Signed-off-by: F.N. Claessen --- .../api/v2_0/tests/test_api_v2_0_users.py | 16 ++++++++-------- .../v2_0/tests/test_api_v2_0_users_fresh_db.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py index f43f6db8e..4569db4e2 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py @@ -22,7 +22,7 @@ def test_get_users_bad_auth(client, use_auth): query = {"account_id": 2} get_users_response = client.get( - url_for("flexmeasures_api_v2_0.get_users"), headers=headers, query_string=query + url_for("UserAPI:get_users"), headers=headers, query_string=query ) print("Server responded with:\n%s" % get_users_response.data) if use_auth: @@ -43,7 +43,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): if include_inactive in (True, False): query["include_inactive"] = include_inactive get_users_response = client.get( - url_for("flexmeasures_api_v2_0.get_users"), query_string=query, headers=headers + url_for("UserAPI:get_users"), query_string=query, headers=headers ) print("Server responded with:\n%s" % get_users_response.json) assert get_users_response.status_code == 200 @@ -71,7 +71,7 @@ def test_get_one_user(client, requesting_user, status_code): headers["Authorization"] = get_auth_token(client, requesting_user, "testtest") get_user_response = client.get( - url_for("flexmeasures_api_v2_0.get_user", id=test_user2_id), + url_for("UserAPI:get_user", id=test_user2_id), headers=headers, ) print("Server responded with:\n%s" % get_user_response.data) @@ -89,7 +89,7 @@ def test_edit_user(client): admin_id = admin.id # without being the user themselves or an admin, the user cannot be edited user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=admin_id), + url_for("UserAPI:patch_user", id=admin_id), headers={ "content-type": "application/json", "Authorization": user2_auth_token, @@ -98,7 +98,7 @@ def test_edit_user(client): ) assert user_edit_response.status_code == 403 user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), + url_for("UserAPI:patch_user", id=user2_id), headers={"content-type": "application/json"}, json={}, ) @@ -107,7 +107,7 @@ def test_edit_user(client): # (id is in the User schema of the API, but we ignore it) headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), + url_for("UserAPI:patch_user", id=user2_id), headers=headers, json={"active": False, "id": 888}, ) @@ -120,7 +120,7 @@ def test_edit_user(client): # admin can edit themselves but not sensitive fields headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=admin_id), + url_for("UserAPI:patch_user", id=admin_id), headers=headers, json={"active": False}, ) @@ -135,7 +135,7 @@ def test_edit_user_with_unexpected_fields(client): with UserContext("test_admin_user@seita.nl") as admin: admin_auth_token = admin.get_auth_token() user_edit_response = client.patch( - url_for("flexmeasures_api_v2_0.patch_user", id=user2_id), + url_for("UserAPI:patch_user", id=user2_id), headers={ "content-type": "application/json", "Authorization": admin_auth_token, diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py index 74bde01f4..1603b8102 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py @@ -28,7 +28,7 @@ def test_user_reset_password(app, client, setup_inactive_user, sender): headers["Authorization"] = (get_auth_token(client, sender, "testtest"),) with app.mail.record_messages() as outbox: pwd_reset_response = client.patch( - url_for("flexmeasures_api_v2_0.reset_user_password", id=user2_id), + url_for("UserAPI:reset_user_password", id=user2_id), query_string={}, headers=headers, ) From c4ea13be60e514f90b71dd6f231f438bfb841649 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 11:10:10 +0100 Subject: [PATCH 52/99] Move user API tests to v3 module Signed-off-by: F.N. Claessen --- flexmeasures/api/v2_0/tests/conftest.py | 19 ------------------- .../api/v3_0/implementations/users.py | 19 +++++++++++-------- flexmeasures/api/v3_0/tests/__init__.py | 0 flexmeasures/api/v3_0/tests/conftest.py | 19 +++++++++++++++++++ .../tests/test_api_v3_0_users.py} | 0 .../tests/test_api_v3_0_users_fresh_db.py} | 0 flexmeasures/ui/crud/users.py | 6 +++--- 7 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 flexmeasures/api/v3_0/tests/__init__.py create mode 100644 flexmeasures/api/v3_0/tests/conftest.py rename flexmeasures/api/{v2_0/tests/test_api_v2_0_users.py => v3_0/tests/test_api_v3_0_users.py} (100%) rename flexmeasures/api/{v2_0/tests/test_api_v2_0_users_fresh_db.py => v3_0/tests/test_api_v3_0_users_fresh_db.py} (100%) diff --git a/flexmeasures/api/v2_0/tests/conftest.py b/flexmeasures/api/v2_0/tests/conftest.py index cc66035be..305a36400 100644 --- a/flexmeasures/api/v2_0/tests/conftest.py +++ b/flexmeasures/api/v2_0/tests/conftest.py @@ -1,5 +1,3 @@ -from flask_security import SQLAlchemySessionUserDatastore -from flask_security.utils import hash_password import pytest @@ -13,20 +11,3 @@ def setup_api_test_data(db, setup_roles_users, add_market_prices, add_battery_as # Add battery asset battery = add_battery_assets["Test battery"] battery.owner = setup_roles_users["Test Prosumer User 2"] - - -@pytest.fixture(scope="module") -def setup_inactive_user(db, setup_accounts, setup_roles_users): - """ - Set up one inactive user. - """ - from flexmeasures.data.models.user import User, Role - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - user_datastore.create_user( - username="inactive test user", - email="inactive@seita.nl", - password=hash_password("testtest"), - account_id=setup_accounts["Prosumer"].id, - active=False, - ) diff --git a/flexmeasures/api/v3_0/implementations/users.py b/flexmeasures/api/v3_0/implementations/users.py index e0e8e289d..3b9f74434 100644 --- a/flexmeasures/api/v3_0/implementations/users.py +++ b/flexmeasures/api/v3_0/implementations/users.py @@ -92,11 +92,11 @@ def reset_password(user_id: int, user: UserModel): db.session.commit() -class UserAPI(FlaskView): - route_base = "/user" +class UsersAPI(FlaskView): + route_base = "/users" + trailing_slash = False - @route("", methods=["GET"]) - def get_users(self): + def index(self): """API endpoint to get users. .. :quickref: User; Download user list @@ -136,8 +136,12 @@ def get_users(self): """ return get() - @route("/", methods=["GET"]) - def get_user(self, id: int): + +class UserAPI(FlaskView): + route_base = "/user" + trailing_slash = False + + def get(self, id: int): """API endpoint to get a user. .. :quickref: User; Get a user @@ -169,8 +173,7 @@ def get_user(self, id: int): """ return fetch_one(id) - @route("/", methods=["PATCH"]) - def patch_user(self, id: int): + def patch(self, id: int): """API endpoint to patch user data. .. :quickref: User; Patch data for an existing user diff --git a/flexmeasures/api/v3_0/tests/__init__.py b/flexmeasures/api/v3_0/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py new file mode 100644 index 000000000..5b1915ff8 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest +from flask_security import SQLAlchemySessionUserDatastore, hash_password + + +@pytest.fixture(scope="module") +def setup_inactive_user(db, setup_accounts, setup_roles_users): + """ + Set up one inactive user. + """ + from flexmeasures.data.models.user import User, Role + + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) + user_datastore.create_user( + username="inactive test user", + email="inactive@seita.nl", + password=hash_password("testtest"), + account_id=setup_accounts["Prosumer"].id, + active=False, + ) \ No newline at end of file diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py similarity index 100% rename from flexmeasures/api/v2_0/tests/test_api_v2_0_users.py rename to flexmeasures/api/v3_0/tests/test_api_v3_0_users.py diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py similarity index 100% rename from flexmeasures/api/v2_0/tests/test_api_v2_0_users_fresh_db.py rename to flexmeasures/api/v3_0/tests/test_api_v3_0_users_fresh_db.py diff --git a/flexmeasures/ui/crud/users.py b/flexmeasures/ui/crud/users.py index c7d4a7501..85049df4b 100644 --- a/flexmeasures/ui/crud/users.py +++ b/flexmeasures/ui/crud/users.py @@ -84,7 +84,7 @@ def index(self): for account in Account.query.all(): get_users_response = InternalApi().get( url_for( - "UserAPI:get_users", + "UsersAPI:index", account_id=account.id, include_inactive=include_inactive, ) @@ -100,7 +100,7 @@ def index(self): @roles_required(ADMIN_ROLE) def get(self, id: str): """GET from /users/""" - get_user_response = InternalApi().get(url_for("UserAPI:get_user", id=id)) + get_user_response = InternalApi().get(url_for("UserAPI:get", id=id)) user: User = process_internal_api_response( get_user_response.json(), make_obj=True ) @@ -117,7 +117,7 @@ def toggle_active(self, id: str): """Toggle activation status via /users/toggle_active/""" user: User = get_user(id) user_response = InternalApi().patch( - url_for("UserAPI:patch_user", id=id), + url_for("UserAPI:patch", id=id), args={"active": not user.active}, ) patched_user: User = process_internal_api_response( From 41371c7d552aebdd3f87203dd327f86439fe58c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 12:10:06 +0100 Subject: [PATCH 53/99] Separate view for user index Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/__init__.py | 3 ++- .../api/v3_0/tests/test_api_v3_0_users.py | 16 ++++++++-------- flexmeasures/ui/tests/test_user_crud.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index c909058fc..dacf37f59 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,7 +1,7 @@ from flask import Flask, Blueprint from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI -from flexmeasures.api.v3_0.implementations.users import UserAPI +from flexmeasures.api.v3_0.implementations.users import UserAPI, UsersAPI def register_at(app: Flask): @@ -11,3 +11,4 @@ def register_at(app: Flask): SensorDataAPI.register(app, route_prefix=v3_0_api_prefix) UserAPI.register(app, route_prefix=v3_0_api_prefix) + UsersAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 4569db4e2..80c4983e0 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -22,7 +22,7 @@ def test_get_users_bad_auth(client, use_auth): query = {"account_id": 2} get_users_response = client.get( - url_for("UserAPI:get_users"), headers=headers, query_string=query + url_for("UsersAPI:index"), headers=headers, query_string=query ) print("Server responded with:\n%s" % get_users_response.data) if use_auth: @@ -43,7 +43,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): if include_inactive in (True, False): query["include_inactive"] = include_inactive get_users_response = client.get( - url_for("UserAPI:get_users"), query_string=query, headers=headers + url_for("UsersAPI:index"), query_string=query, headers=headers ) print("Server responded with:\n%s" % get_users_response.json) assert get_users_response.status_code == 200 @@ -71,7 +71,7 @@ def test_get_one_user(client, requesting_user, status_code): headers["Authorization"] = get_auth_token(client, requesting_user, "testtest") get_user_response = client.get( - url_for("UserAPI:get_user", id=test_user2_id), + url_for("UserAPI:get", id=test_user2_id), headers=headers, ) print("Server responded with:\n%s" % get_user_response.data) @@ -89,7 +89,7 @@ def test_edit_user(client): admin_id = admin.id # without being the user themselves or an admin, the user cannot be edited user_edit_response = client.patch( - url_for("UserAPI:patch_user", id=admin_id), + url_for("UserAPI:patch", id=admin_id), headers={ "content-type": "application/json", "Authorization": user2_auth_token, @@ -98,7 +98,7 @@ def test_edit_user(client): ) assert user_edit_response.status_code == 403 user_edit_response = client.patch( - url_for("UserAPI:patch_user", id=user2_id), + url_for("UserAPI:patch", id=user2_id), headers={"content-type": "application/json"}, json={}, ) @@ -107,7 +107,7 @@ def test_edit_user(client): # (id is in the User schema of the API, but we ignore it) headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( - url_for("UserAPI:patch_user", id=user2_id), + url_for("UserAPI:patch", id=user2_id), headers=headers, json={"active": False, "id": 888}, ) @@ -120,7 +120,7 @@ def test_edit_user(client): # admin can edit themselves but not sensitive fields headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( - url_for("UserAPI:patch_user", id=admin_id), + url_for("UserAPI:patch", id=admin_id), headers=headers, json={"active": False}, ) @@ -135,7 +135,7 @@ def test_edit_user_with_unexpected_fields(client): with UserContext("test_admin_user@seita.nl") as admin: admin_auth_token = admin.get_auth_token() user_edit_response = client.patch( - url_for("UserAPI:patch_user", id=user2_id), + url_for("UserAPI:patch", id=user2_id), headers={ "content-type": "application/json", "Authorization": admin_auth_token, diff --git a/flexmeasures/ui/tests/test_user_crud.py b/flexmeasures/ui/tests/test_user_crud.py index a2a698449..5882152b0 100644 --- a/flexmeasures/ui/tests/test_user_crud.py +++ b/flexmeasures/ui/tests/test_user_crud.py @@ -23,7 +23,7 @@ def test_user_crud_as_non_admin(client, as_prosumer_user1, view): def test_user_list(client, as_admin, requests_mock): requests_mock.get( - "http://localhost//api/v3_0/user", + "http://localhost//api/v3_0/users", status_code=200, json=mock_user_response(multiple=True), ) From d2a2e5f0d582efbaca034584710d4755261544c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 12:13:22 +0100 Subject: [PATCH 54/99] Fix test Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 5b1915ff8..edb01c9c1 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -2,6 +2,14 @@ from flask_security import SQLAlchemySessionUserDatastore, hash_password +@pytest.fixture(scope="module", autouse=True) +def setup_api_test_data(db, setup_roles_users): + """ + Set up data for API v3.0 tests. + """ + print("Setting up data for API v3.0 tests on %s" % db.engine) + + @pytest.fixture(scope="module") def setup_inactive_user(db, setup_accounts, setup_roles_users): """ From 0c2d199cf285761bc6311c7c88f9e0157f0ab886 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 12:15:17 +0100 Subject: [PATCH 55/99] Update documentation tutorials to v3 Signed-off-by: F.N. Claessen --- documentation/getting-started.rst | 4 ++-- documentation/tut/posting_data.rst | 4 ++-- documentation/tut/toy-example-from-scratch.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 923b9b2f1..f7fdd320a 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -147,7 +147,7 @@ For the account ID, I looked at the output of ``flexmeasures add account`` (the The second way to add an asset is the UI ― head over to ``https://localhost:5000/assets`` (after you started FlexMeasures, see step "Run FlexMeasures" further down) and add a new asset there in a web form. -Finally, you can also use the `POST /api/v2_0/assets `_ endpoint in the FlexMeasures API to create an asset. +Finally, you can also use the `POST /api/v3_0/assets `_ endpoint in the FlexMeasures API to create an asset. Add your first sensor @@ -195,7 +195,7 @@ First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` This assumes you have a file `my-data.csv` with measurements, which was exported from some legacy database, and that the data is about our sensor with ID 1. This command has many options, so do use its ``--help`` function. -Second, you can use the `POST /api/v2_0/postSensorData `_ endpoint in the FlexMeasures API to send meter data. +Second, you can use the `POST /api/v3_0/sensorData `_ endpoint in the FlexMeasures API to send meter data. Finally, you can tell FlexMeasures to create forecasts for your meter data with the ``flexmeasures add forecasts`` command, here is an example: diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index d8ea63ac6..00817425e 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -31,7 +31,7 @@ Prerequisites Posting sensor data ------------------- -Sensor data (both observations and forecasts) can be posted to `POST /api/v2_0/postSensorData <../api/v2_0.html#post--api-v2_0-postSensorData>`_. +Sensor data (both observations and forecasts) can be posted to `POST /api/v3_0/sensorData <../api/v3_0.html#post--api-v3_0-sensorData>`_. This endpoint represents the basic method of getting time series data into FlexMeasures via API. It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that have happened in the past or will happen in the future. Some examples: @@ -270,7 +270,7 @@ Posting flexibility states There is one more crucial kind of data that FlexMeasures needs to know about: What are the current states of flexible devices? For example, a battery has a state of charge. The USEF framework defines a so-called "UDI-Event" (UDI stands for Universal Device Interface) to communicate settings for devices with Active Demand & Supply (ADS). -Owners of such devices can post these states to `POST /api/v2_0/postUdiEvent <../api/v2_0.html#post--api-v2_0-postUdiEvent>`_. The URL might look like this: +Owners of such devices can post these states to `POST /api/v3_0/postUdiEvent <../api/v3_0.html#post--api-v3_0-postUdiEvent>`_. The URL might look like this: .. code-block:: html diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index a1d17a2fb..bba832372 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -108,7 +108,7 @@ If you want, you can inspect what you created: Yes, that is quite a large battery :) -.. note:: Obviously, you can use the ``flexmeasures`` command to create your own, custom account and assets. See :ref:`cli`. And to create, edit or read asset data via the API, see :ref:`v2_0`. +.. note:: Obviously, you can use the ``flexmeasures`` command to create your own, custom account and assets. See :ref:`cli`. And to create, edit or read asset data via the API, see :ref:`v3_0`. We can also look at the battery asset in the UI of FlexMeasures (start FlexMeasures with ``flexmeasures run``, username is "toy-user@flexmeasures.io", password is "toy-password"): From 482cde94cc70c27502084ee64d515417a212c84d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 12:16:48 +0100 Subject: [PATCH 56/99] No trailing slash Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/implementations/sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/v3_0/implementations/sensor_data.py b/flexmeasures/api/v3_0/implementations/sensor_data.py index e1fa5023a..b3c516362 100644 --- a/flexmeasures/api/v3_0/implementations/sensor_data.py +++ b/flexmeasures/api/v3_0/implementations/sensor_data.py @@ -15,6 +15,7 @@ class SensorDataAPI(FlaskView): route_base = "/sensorData" + trailing_slash = False decorators = [auth_required()] @route("/", methods=["POST"]) From 820521eee8e1ee3c28b4426f845e95c73bcb9c3f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 12:31:16 +0100 Subject: [PATCH 57/99] Make `type` field in v3 Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 14 +++++++++----- .../api/v3_0/implementations/sensor_data.py | 4 ---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7641a9edb..5a4cc0f04 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -96,8 +96,10 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): + + # Optional field that can be used for extra validation type = fields.Str( - required=True, + required=False, validate=OneOf( [ "GetSensorDataRequest", @@ -115,7 +117,7 @@ def check_user_may_read(self, data, **kwargs): @validates_schema def check_schema_unit_against_type(self, data, **kwargs): requested_unit = data["unit"] - _type = data["type"] + _type = data.get("type", None) if ( _type in ( @@ -153,7 +155,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: # Post-load configuration of belief timing against message type horizons_at_least = sensor_data_description.get("horizon", None) horizons_at_most = None - _type = sensor_data_description["type"] + _type = sensor_data_description.get("type", None) if _type == "PostMeterDataRequest": horizons_at_most = timedelta(0) elif _type == "PostPrognosisRequest": @@ -211,7 +213,9 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): (sets a resolution parameter which we can pass to the data collection function). """ + # Optional field that can be used for extra validation type = fields.Str( + required=False, validate=OneOf( [ "PostSensorDataRequest", @@ -235,7 +239,7 @@ def check_user_may_create(self, data, **kwargs): @validates_schema def check_schema_unit_against_type(self, data, **kwargs): posted_unit = data["unit"] - _type = data["type"] + _type = data.get("type", None) if ( _type in ( @@ -272,7 +276,7 @@ def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: bdf = self.load_bdf(data) # Post-load validation against message type - _type = data["type"] + _type = data.get("type", None) if _type == "PostMeterDataRequest": if any(h > timedelta(0) for h in bdf.belief_horizons): raise ValidationError("Meter data must lie in the past.") diff --git a/flexmeasures/api/v3_0/implementations/sensor_data.py b/flexmeasures/api/v3_0/implementations/sensor_data.py index b3c516362..1b5dfa02e 100644 --- a/flexmeasures/api/v3_0/implementations/sensor_data.py +++ b/flexmeasures/api/v3_0/implementations/sensor_data.py @@ -34,7 +34,6 @@ def post(self, bdf: BeliefsDataFrame): .. code-block:: json { - "type": "PostSensorDataRequest", "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", "values": [-11.28, -11.28, -11.28, -11.28], "start": "2021-06-07T00:00:00+02:00", @@ -51,7 +50,6 @@ def post(self, bdf: BeliefsDataFrame): FlexMeasures will attempt to upsample lower resolutions. """ response, code = save_and_enqueue(bdf) - response.update(type="PostSensorDataResponse") return response, code @route("/", methods=["GET"]) @@ -69,7 +67,6 @@ def get(self, response: dict): .. code-block:: json { - "type": "GetSensorDataRequest", "sensor": "ea1.2021-01.io.flexmeasures:fm1.1", "start": "2021-06-07T00:00:00+02:00", "duration": "PT1H", @@ -78,5 +75,4 @@ def get(self, response: dict): The unit has to be convertible from the sensor's unit. """ - response.update(type="GetSensorDataResponse") return json.dumps(response) From 0fc14b6e6d95a0f4c5470373659212d9e7a9da77 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 13:05:26 +0100 Subject: [PATCH 58/99] Add v3 to API index Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 6 +++--- flexmeasures/api/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index be573f97a..f4bd88061 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -40,10 +40,10 @@ Let's see what the ``/api`` endpoint returns: >>> import requests >>> res = requests.get("https://company.flexmeasures.io/api") >>> res.json() - {'flexmeasures_version': '0.4.0', - 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: /api/requestAuthToken', + {'flexmeasures_version': '0.9.0', + 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: /api/requestAuthToken', 'status': 200, - 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0'] + 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0', 'v3_0'] } So this tells us which API versions exist. For instance, we know that the latest API version is available at diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 9833ffd5e..dd7164f4e 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -73,9 +73,9 @@ def get_versions() -> dict: """ response = { "message": "For these API versions a public endpoint is available, listing its service. For example: " - "/api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: " + "/api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: " "/api/requestAuthToken", - "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0"], + "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0", "v3_0"], "flexmeasures_version": flexmeasures_version, } return response From 70a940684026b02f6fe3c7b0ca91a2fe7beddf9c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 13:17:11 +0100 Subject: [PATCH 59/99] Update introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index f4bd88061..02bb6528b 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -23,13 +23,13 @@ So if you are running FlexMeasures on your computer, it would be: https://localhost:5000/api -At Seita, we run servers for our clients at: +Let's assume we are running a server for a client at: .. code-block:: html https://company.flexmeasures.io/api -where `company` is a hosting customer of ours. All their accounts' data lives on that server. +where `company` is a client of ours. All their accounts' data lives on that server. We assume in this document that the FlexMeasures instance you want to connect to is hosted at https://company.flexmeasures.io. @@ -46,11 +46,11 @@ Let's see what the ``/api`` endpoint returns: 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0', 'v3_0'] } -So this tells us which API versions exist. For instance, we know that the latest API version is available at +So this tells us which API versions exist. For instance, we know that the latest API version is available at: .. code-block:: html - https://company.flexmeasures.io/api/v2_0 + https://company.flexmeasures.io/api/v3_0 Also, we can see that a list of endpoints which are available at (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. From 281af3bae165d42bed2d0737b02ffe849a4596d8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 13:32:04 +0100 Subject: [PATCH 60/99] Fix comment about where to request auth token Signed-off-by: F.N. Claessen --- documentation/tut/building_uis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst index 08856e594..2041b943c 100644 --- a/documentation/tut/building_uis.rst +++ b/documentation/tut/building_uis.rst @@ -20,7 +20,7 @@ This tutorial will show how the FlexMeasures API can be used from JavaScript to Get an authentication token ----------------------- -FlexMeasures provides the `POST /api/v2_0/requestAuthToken <../api/v2_0.html#post--api-v2_0-requestAuthToken>`_ endpoint, as discussed in :ref:`api_auth`. +FlexMeasures provides the `POST /api/requestAuthToken <../api/v2_0.html#post--api-v2_0-requestAuthToken>`_ endpoint, as discussed in :ref:`api_auth`. Here is a JavaScript function to call it: .. code-block:: JavaScript From d14d85e6e67719d294b9a1a7e3ca5cfc15e18b7a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 13:32:45 +0100 Subject: [PATCH 61/99] flake8 Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 2 +- flexmeasures/api/v3_0/__init__.py | 2 +- flexmeasures/api/v3_0/tests/conftest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 5a4cc0f04..a85c300cf 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -4,7 +4,7 @@ from flask_login import current_user from isodate import datetime_isoformat, duration_isoformat from marshmallow import fields, post_load, validates_schema, ValidationError -from marshmallow.validate import Equal, OneOf +from marshmallow.validate import OneOf from marshmallow_polyfield import PolyField from timely_beliefs import BeliefsDataFrame import pandas as pd diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index dacf37f59..c7d933adb 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,4 +1,4 @@ -from flask import Flask, Blueprint +from flask import Flask from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI from flexmeasures.api.v3_0.implementations.users import UserAPI, UsersAPI diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index edb01c9c1..be983fd5a 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -24,4 +24,4 @@ def setup_inactive_user(db, setup_accounts, setup_roles_users): password=hash_password("testtest"), account_id=setup_accounts["Prosumer"].id, active=False, - ) \ No newline at end of file + ) From 89e8dffb1e97ffecbded6bfbdc2b1c4315a9111e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 13:55:42 +0100 Subject: [PATCH 62/99] black Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index a85c300cf..e3b1dc241 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -107,7 +107,7 @@ class GetSensorDataSchema(SensorDataDescriptionSchema): "GetPrognosisRequest", "GetPriceDataRequest", ] - ) + ), ) @validates_schema @@ -118,14 +118,10 @@ def check_user_may_read(self, data, **kwargs): def check_schema_unit_against_type(self, data, **kwargs): requested_unit = data["unit"] _type = data.get("type", None) - if ( - _type - in ( - "GetMeterDataRequest", - "GetPrognosisRequest", - ) - and not units_are_convertible(requested_unit, "MW") - ): + if _type in ( + "GetMeterDataRequest", + "GetPrognosisRequest", + ) and not units_are_convertible(requested_unit, "MW"): raise ValidationError( f"The unit requested for this message type should be convertible from MW, got incompatible unit: {requested_unit}" ) @@ -224,7 +220,7 @@ class PostSensorDataSchema(SensorDataDescriptionSchema): "PostPriceDataRequest", "PostWeatherDataRequest", ] - ) + ), ) values = PolyField( deserialization_schema_selector=select_schema_to_ensure_list_of_floats, @@ -240,14 +236,10 @@ def check_user_may_create(self, data, **kwargs): def check_schema_unit_against_type(self, data, **kwargs): posted_unit = data["unit"] _type = data.get("type", None) - if ( - _type - in ( - "PostMeterDataRequest", - "PostPrognosisRequest", - ) - and not units_are_convertible(posted_unit, "MW") - ): + if _type in ( + "PostMeterDataRequest", + "PostPrognosisRequest", + ) and not units_are_convertible(posted_unit, "MW"): raise ValidationError( f"The unit required for this message type should be convertible to MW, got incompatible unit: {posted_unit}" ) From 2f2cee0fe556e47dc4c786f6b49604c71a125957 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 16:52:05 +0100 Subject: [PATCH 63/99] Revert some documentation changes that are ahead of code changes Signed-off-by: F.N. Claessen --- documentation/getting-started.rst | 2 +- documentation/tut/forecasting_scheduling.rst | 2 +- documentation/tut/posting_data.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index f7fdd320a..7c24f6783 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -147,7 +147,7 @@ For the account ID, I looked at the output of ``flexmeasures add account`` (the The second way to add an asset is the UI ― head over to ``https://localhost:5000/assets`` (after you started FlexMeasures, see step "Run FlexMeasures" further down) and add a new asset there in a web form. -Finally, you can also use the `POST /api/v3_0/assets `_ endpoint in the FlexMeasures API to create an asset. +Finally, you can also use the `POST /api/v2_0/assets `_ endpoint in the FlexMeasures API to create an asset. Add your first sensor diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 76ec02164..7c51073d0 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -116,7 +116,7 @@ Getting power forecasts (prognoses) Prognoses (the USEF term used for power forecasts) are used by FlexMeasures to determine the best control signals to valorise on balancing opportunities. -You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. +You can access forecasts via the FlexMeasures API at `GET /api/v2_0/getPrognosis <../api/v2_0.html#get--api-v2_0-getPrognosis>`_. Getting them might be useful if you want to use prognoses in your own system, or to check their accuracy against meter data, i.e. the realised power measurements. The FlexMeasures UI also lists forecast accuracy, and visualises prognoses and meter data next to each other. diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 00817425e..a18f0b6b3 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -270,7 +270,7 @@ Posting flexibility states There is one more crucial kind of data that FlexMeasures needs to know about: What are the current states of flexible devices? For example, a battery has a state of charge. The USEF framework defines a so-called "UDI-Event" (UDI stands for Universal Device Interface) to communicate settings for devices with Active Demand & Supply (ADS). -Owners of such devices can post these states to `POST /api/v3_0/postUdiEvent <../api/v3_0.html#post--api-v3_0-postUdiEvent>`_. The URL might look like this: +Owners of such devices can post these states to `POST /api/v2_0/postUdiEvent <../api/v2_0.html#post--api-v2_0-postUdiEvent>`_. The URL might look like this: .. code-block:: html From abfc41ac398071b81b5d29f9d190b5a9ed17cc5e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 16:56:13 +0100 Subject: [PATCH 64/99] Use one common base route Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/__init__.py | 3 +-- flexmeasures/api/v3_0/implementations/users.py | 9 ++------- flexmeasures/api/v3_0/tests/test_api_v3_0_users.py | 4 ++-- flexmeasures/ui/crud/users.py | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index c7d933adb..c5a437747 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,7 +1,7 @@ from flask import Flask from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI -from flexmeasures.api.v3_0.implementations.users import UserAPI, UsersAPI +from flexmeasures.api.v3_0.implementations.users import UserAPI def register_at(app: Flask): @@ -11,4 +11,3 @@ def register_at(app: Flask): SensorDataAPI.register(app, route_prefix=v3_0_api_prefix) UserAPI.register(app, route_prefix=v3_0_api_prefix) - UsersAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/implementations/users.py b/flexmeasures/api/v3_0/implementations/users.py index 3b9f74434..c3993a838 100644 --- a/flexmeasures/api/v3_0/implementations/users.py +++ b/flexmeasures/api/v3_0/implementations/users.py @@ -92,8 +92,8 @@ def reset_password(user_id: int, user: UserModel): db.session.commit() -class UsersAPI(FlaskView): - route_base = "/users" +class UserAPI(FlaskView): + route_base = "/user" trailing_slash = False def index(self): @@ -136,11 +136,6 @@ def index(self): """ return get() - -class UserAPI(FlaskView): - route_base = "/user" - trailing_slash = False - def get(self, id: int): """API endpoint to get a user. diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 80c4983e0..1b7314f3d 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -22,7 +22,7 @@ def test_get_users_bad_auth(client, use_auth): query = {"account_id": 2} get_users_response = client.get( - url_for("UsersAPI:index"), headers=headers, query_string=query + url_for("UserAPI:index"), headers=headers, query_string=query ) print("Server responded with:\n%s" % get_users_response.data) if use_auth: @@ -43,7 +43,7 @@ def test_get_users_inactive(client, setup_inactive_user, include_inactive): if include_inactive in (True, False): query["include_inactive"] = include_inactive get_users_response = client.get( - url_for("UsersAPI:index"), query_string=query, headers=headers + url_for("UserAPI:index"), query_string=query, headers=headers ) print("Server responded with:\n%s" % get_users_response.json) assert get_users_response.status_code == 200 diff --git a/flexmeasures/ui/crud/users.py b/flexmeasures/ui/crud/users.py index 85049df4b..5b4785f2f 100644 --- a/flexmeasures/ui/crud/users.py +++ b/flexmeasures/ui/crud/users.py @@ -84,7 +84,7 @@ def index(self): for account in Account.query.all(): get_users_response = InternalApi().get( url_for( - "UsersAPI:index", + "UserAPI:index", account_id=account.id, include_inactive=include_inactive, ) From 3fec5829d28d051d0dd029b4d6a6538a7a1295fd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 16:58:14 +0100 Subject: [PATCH 65/99] Prefer plural base route Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/implementations/users.py | 2 +- flexmeasures/ui/tests/test_user_crud.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/implementations/users.py b/flexmeasures/api/v3_0/implementations/users.py index c3993a838..99a6c62ca 100644 --- a/flexmeasures/api/v3_0/implementations/users.py +++ b/flexmeasures/api/v3_0/implementations/users.py @@ -93,7 +93,7 @@ def reset_password(user_id: int, user: UserModel): class UserAPI(FlaskView): - route_base = "/user" + route_base = "/users" trailing_slash = False def index(self): diff --git a/flexmeasures/ui/tests/test_user_crud.py b/flexmeasures/ui/tests/test_user_crud.py index 5882152b0..c4506fa5f 100644 --- a/flexmeasures/ui/tests/test_user_crud.py +++ b/flexmeasures/ui/tests/test_user_crud.py @@ -37,7 +37,7 @@ def test_user_list(client, as_admin, requests_mock): def test_user_page(client, as_admin, requests_mock): mock_user = mock_user_response(as_list=False) requests_mock.get( - "http://localhost//api/v3_0/user/2", status_code=200, json=mock_user + "http://localhost//api/v3_0/users/2", status_code=200, json=mock_user ) requests_mock.get( "http://localhost//api/v2_0/assets", @@ -55,7 +55,7 @@ def test_deactivate_user(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v3_0/user/{user2.id}", + f"http://localhost//api/v3_0/users/{user2.id}", status_code=200, json={"active": False}, ) @@ -72,7 +72,7 @@ def test_reset_password(client, as_admin, requests_mock): """Test it does not fail (logic is tested in API tests) and displays an answer.""" user2 = find_user_by_email("test_prosumer_user_2@seita.nl", keep_in_session=False) requests_mock.patch( - f"http://localhost//api/v3_0/user/{user2.id}/password-reset", + f"http://localhost//api/v3_0/users/{user2.id}/password-reset", status_code=200, ) user_page = client.get( From 80ed32d2417ae28421935ae3f28ae9ba67313c07 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 14 Mar 2022 17:06:32 +0100 Subject: [PATCH 66/99] different shade of black Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index e3b1dc241..089523864 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -118,10 +118,14 @@ def check_user_may_read(self, data, **kwargs): def check_schema_unit_against_type(self, data, **kwargs): requested_unit = data["unit"] _type = data.get("type", None) - if _type in ( - "GetMeterDataRequest", - "GetPrognosisRequest", - ) and not units_are_convertible(requested_unit, "MW"): + if ( + _type + in ( + "GetMeterDataRequest", + "GetPrognosisRequest", + ) + and not units_are_convertible(requested_unit, "MW") + ): raise ValidationError( f"The unit requested for this message type should be convertible from MW, got incompatible unit: {requested_unit}" ) @@ -236,10 +240,14 @@ def check_user_may_create(self, data, **kwargs): def check_schema_unit_against_type(self, data, **kwargs): posted_unit = data["unit"] _type = data.get("type", None) - if _type in ( - "PostMeterDataRequest", - "PostPrognosisRequest", - ) and not units_are_convertible(posted_unit, "MW"): + if ( + _type + in ( + "PostMeterDataRequest", + "PostPrognosisRequest", + ) + and not units_are_convertible(posted_unit, "MW") + ): raise ValidationError( f"The unit required for this message type should be convertible to MW, got incompatible unit: {posted_unit}" ) From a26fad16903181baea3eb1a3d4ec1ef652cbe5c0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 13:54:41 +0100 Subject: [PATCH 67/99] typos Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 7601753a8..80c9908f1 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -80,7 +80,7 @@ v1.3-8 | 2020-04-02 *Affects all versions since v1.0*. -- [**Breaking change**, partially reverted in v1.3-9] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API version below v2.0. +- [**Breaking change**, partially reverted in v1.3-9] Deprecated the automatic inference of horizons for *postMeterData*, *postPrognosis*, *postPriceData* and *postWeatherData* endpoints for API versions below v2.0. v1.3-7 | 2020-12-16 """"""""""""""""""" @@ -107,7 +107,7 @@ v1.3-5 | 2020-10-29 - Endpoints to POST meter data will now check incoming data to see if the required asset's resolution is being used ― upsampling is done if possible. These endpoints can now return the REQUIRED_INFO_MISSING status 400 response. - Endpoints to GET meter data will return data in the asset's resolution ― downsampling to the "resolution" field is done if possible. -- As they need to determine the asset, all of the mentioned POST and GET endpoints can now return the UNRECOGNIZED_ASSET status 4000 response. +- As they need to determine the asset, all of the mentioned POST and GET endpoints can now return the UNRECOGNIZED_ASSET status 400 response. v1.3-4 | 2020-06-18 """"""""""""""""""" From 6f34010859a2a3c13fe7138d2506f529c07c280d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 15:23:33 +0100 Subject: [PATCH 68/99] Transition SensorDataAPI:post to SensorAPI:post_data and SensorDataAPI:get to SensorAPI:get_data Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/tests/test_sensor_data.py | 10 +++++----- .../api/dev/tests/test_sensor_data_fresh_db.py | 2 +- flexmeasures/api/v3_0/__init__.py | 4 ++-- flexmeasures/api/v3_0/implementations/sensor_data.py | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/dev/tests/test_sensor_data.py index 2f1eb1124..5e12b899c 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data.py +++ b/flexmeasures/api/dev/tests/test_sensor_data.py @@ -21,7 +21,7 @@ def test_post_sensor_data_bad_auth(client, setup_api_test_data, use_auth): post_data = make_sensor_data_request_for_gas_sensor() post_data_response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), headers=headers, json=post_data, ) @@ -55,7 +55,7 @@ def test_post_invalid_sensor_data( # this guy is allowed to post sensorData auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) @@ -70,7 +70,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that 1st time posting the data succeeds response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) @@ -79,7 +79,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that 2nd time posting the same data succeeds informatively response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) @@ -90,7 +90,7 @@ def test_post_sensor_data_twice(client, setup_api_test_data): # Check that replacing data fails informatively post_data["values"][0] = 100 response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py index f8f878a63..56d2af2be 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py @@ -39,7 +39,7 @@ def test_post_sensor_data( auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest") response = client.post( - url_for("SensorDataAPI:post"), + url_for("SensorAPI:post_data"), json=post_data, headers={"Authorization": auth_token}, ) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index c5a437747..19758fccf 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,6 +1,6 @@ from flask import Flask -from flexmeasures.api.v3_0.implementations.sensor_data import SensorDataAPI +from flexmeasures.api.v3_0.implementations.sensor_data import SensorAPI from flexmeasures.api.v3_0.implementations.users import UserAPI @@ -9,5 +9,5 @@ def register_at(app: Flask): v3_0_api_prefix = "/api/v3_0" - SensorDataAPI.register(app, route_prefix=v3_0_api_prefix) + SensorAPI.register(app, route_prefix=v3_0_api_prefix) UserAPI.register(app, route_prefix=v3_0_api_prefix) diff --git a/flexmeasures/api/v3_0/implementations/sensor_data.py b/flexmeasures/api/v3_0/implementations/sensor_data.py index 1b5dfa02e..3e173c2e5 100644 --- a/flexmeasures/api/v3_0/implementations/sensor_data.py +++ b/flexmeasures/api/v3_0/implementations/sensor_data.py @@ -12,18 +12,18 @@ from flexmeasures.api.common.utils.api_utils import save_and_enqueue -class SensorDataAPI(FlaskView): +class SensorAPI(FlaskView): - route_base = "/sensorData" + route_base = "/sensors" trailing_slash = False decorators = [auth_required()] - @route("/", methods=["POST"]) + @route("/data", methods=["POST"]) @use_args( PostSensorDataSchema(), location="json", ) - def post(self, bdf: BeliefsDataFrame): + def post_data(self, bdf: BeliefsDataFrame): """ Post sensor data to FlexMeasures. @@ -52,12 +52,12 @@ def post(self, bdf: BeliefsDataFrame): response, code = save_and_enqueue(bdf) return response, code - @route("/", methods=["GET"]) + @route("/data", methods=["GET"]) @use_args( GetSensorDataSchema(), location="query", ) - def get(self, response: dict): + def get_data(self, response: dict): """Get sensor data from FlexMeasures. .. :quickref: Data; Download sensor data From 0f2a8f0a124f1fd6b116311b8c3bd42c33bf73e2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 15:24:05 +0100 Subject: [PATCH 69/99] Rename module Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/__init__.py | 2 +- .../api/v3_0/implementations/{sensor_data.py => sensors.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename flexmeasures/api/v3_0/implementations/{sensor_data.py => sensors.py} (100%) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index 19758fccf..51cf9b134 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,6 +1,6 @@ from flask import Flask -from flexmeasures.api.v3_0.implementations.sensor_data import SensorAPI +from flexmeasures.api.v3_0.implementations.sensors import SensorAPI from flexmeasures.api.v3_0.implementations.users import UserAPI diff --git a/flexmeasures/api/v3_0/implementations/sensor_data.py b/flexmeasures/api/v3_0/implementations/sensors.py similarity index 100% rename from flexmeasures/api/v3_0/implementations/sensor_data.py rename to flexmeasures/api/v3_0/implementations/sensors.py From bf277b06e98a22840cec3b65932331fd9ed7b782 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 15:31:38 +0100 Subject: [PATCH 70/99] Update documentation for moving from SensorDataAPI:post to SensorAPI:post_data Signed-off-by: F.N. Claessen --- documentation/api/v3_0.rst | 4 ++-- documentation/getting-started.rst | 2 +- documentation/tut/posting_data.rst | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/documentation/api/v3_0.rst b/documentation/api/v3_0.rst index c441b01d9..0d589fb4a 100644 --- a/documentation/api/v3_0.rst +++ b/documentation/api/v3_0.rst @@ -7,7 +7,7 @@ Summary ------- .. qrefflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.implementations.sensor_data, flexmeasures.api.v3_0.implementations.users + :modules: flexmeasures.api.v3_0.implementations.sensors, flexmeasures.api.v3_0.implementations.users :order: path :include-empty-docstring: @@ -15,6 +15,6 @@ API Details ----------- .. autoflask:: flexmeasures.app:create(env="documentation") - :modules: flexmeasures.api.v3_0.implementations.sensor_data, flexmeasures.api.v3_0.implementations.users + :modules: flexmeasures.api.v3_0.implementations.sensors, flexmeasures.api.v3_0.implementations.users :order: path :include-empty-docstring: diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 7c24f6783..ef0442794 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -195,7 +195,7 @@ First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` This assumes you have a file `my-data.csv` with measurements, which was exported from some legacy database, and that the data is about our sensor with ID 1. This command has many options, so do use its ``--help`` function. -Second, you can use the `POST /api/v3_0/sensorData `_ endpoint in the FlexMeasures API to send meter data. +Second, you can use the `POST /api/v3_0/sensors/data `_ endpoint in the FlexMeasures API to send meter data. Finally, you can tell FlexMeasures to create forecasts for your meter data with the ``flexmeasures add forecasts`` command, here is an example: diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index a18f0b6b3..60e21b161 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -31,7 +31,7 @@ Prerequisites Posting sensor data ------------------- -Sensor data (both observations and forecasts) can be posted to `POST /api/v3_0/sensorData <../api/v3_0.html#post--api-v3_0-sensorData>`_. +Sensor data (both observations and forecasts) can be posted to `POST /api/v3_0/sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_. This endpoint represents the basic method of getting time series data into FlexMeasures via API. It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that have happened in the past or will happen in the future. Some examples: @@ -46,7 +46,7 @@ The exact URL will depend on your domain name, and will look approximately like .. code-block:: html - [POST] https://company.flexmeasures.io/api//sensorData + [POST] https://company.flexmeasures.io/api//sensors/data This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. @@ -98,7 +98,7 @@ Posting power data ------------------ For power data, USEF specifies separate message types for observations and forecasts. -Correspondingly, we allow the following message types to be used with the [POST] /sensorData endpoint (see :ref:`posting_sensor_data`): +Correspondingly, we allow the following message types to be used with the [POST] /sensors/data endpoint (see :ref:`posting_sensor_data`): .. code-block:: json From dcb46f3d12f79082fdc6f624ad6e939acc3b492a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 15:47:54 +0100 Subject: [PATCH 71/99] Update introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 02bb6528b..414b58eee 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -374,10 +374,10 @@ Technically, this is equal to: This intuitive convention allows us to reduce communication by sending univariate timeseries as arrays. -Notation for v1 -""""""""""""""" +Notation for v1, v2 and v3 +"""""""""""""""""""""""""" -For version 1 and 2 of the API, only equidistant timeseries data is expected to be communicated. Therefore: +For version 1, 2 and 3 of the API, only equidistant timeseries data is expected to be communicated. Therefore: - only the array notation should be used (first notation from above), - "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and @@ -496,19 +496,25 @@ Resolutions Specifying a resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the resolution by dividing the duration by the number of values. -When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the assets which are posted to. If these can't be matched (through upsampling), an error will occur. +When POSTing data, FlexMeasures checks this computed resolution against the required resolution of the sensors which are posted to. If these can't be matched (through upsampling), an error will occur. GET requests (such as *getMeterData*) return data in the resolution which the sensor is configured for. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. The specified resolution needs to be a multiple -of the asset's resolution, e.g. hourly or daily values if the asset's resolution is 15 minutes. +of the sensor's resolution, e.g. hourly or daily values if the sensor's resolution is 15 minutes. .. _units: Units ^^^^^ -Valid units for timeseries data in version 1 of the API are "MW" only. +From API version 3 onwards, we are much more flexible with sent units. +A valid unit for timeseries data is any unit that is convertible to the configured sensor unit registered in FlexMeasures. +So, for example, you can send timeseries data with "W" unit to a "kW" sensor. +And if you wish to do so, you can even send a timeseries with "kWh" unit to a "kW" sensor. +In this case, FlexMeasures will convert the data using the resolution of the timeseries. + +For API versions 1 and 2, the unit sent needs to be an exact match with the sensor unit, and only "MW" is allowed for power sensors. .. _signs: From d3924239100601b286d6f2fe7e5c2f5b9006ed47 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 15:54:08 +0100 Subject: [PATCH 72/99] API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 80c9908f1..ff61e6b6c 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,23 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-0 | 2022-03-15 +""""""""""""""""""" + +- Added REST endpoint for managing sensor data: `/sensors/data` (GET, POST). +- [**Breaking change**] Switched to plural resource names for REST endpoints: `/users/` (GET, PATCH) and `/users//password-reset` (PATCH). +- [**Breaking change**] Deprecated the following endpoints: + + - *getConnection* + - *getMeterData* -> use `/sensors/data` (GET) instead + - *getPrognosis* -> use `/sensors/data` (GET) instead + - *getService* + - *postMeterData* -> use `/sensors/data` (POST) instead + - *postPriceData* -> use `/sensors/data` (POST) instead + - *postPrognosis* -> use `/sensors/data` (POST) instead + - *postWeatherData* -> use `/sensors/data` (POST) instead + - *restoreData* + v2.0-4 | 2022-01-04 """"""""""""""""""" @@ -42,12 +59,12 @@ v2.0-2 | 2021-04-02 v2.0-1 | 2021-02-19 """"""""""""""""""" -- REST endpoints for managing users: `/users/` (GET), `/user/` (GET, PATCH) and `/user//password-reset` (PATCH). +- Added REST endpoints for managing users: `/users/` (GET), `/user/` (GET, PATCH) and `/user//password-reset` (PATCH). v2.0-0 | 2020-11-14 """"""""""""""""""" -- REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). +- Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). v1.3-11 | 2022-01-05 From 64e11001bcaf25bb6d4b6a1bce7b0937a9267f7b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 17:09:05 +0100 Subject: [PATCH 73/99] Implement /sensors [GET] to replace getConnection Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 3 +- .../api/v3_0/implementations/sensors.py | 33 ++++++++++++++++++- flexmeasures/data/schemas/sensors.py | 1 + flexmeasures/data/services/sensors.py | 26 +++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 flexmeasures/data/services/sensors.py diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index ff61e6b6c..e07eaf740 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,11 +9,12 @@ API change log v3.0-0 | 2022-03-15 """"""""""""""""""" +- Added REST endpoint for listing sensors: `/sensors` (GET). - Added REST endpoint for managing sensor data: `/sensors/data` (GET, POST). - [**Breaking change**] Switched to plural resource names for REST endpoints: `/users/` (GET, PATCH) and `/users//password-reset` (PATCH). - [**Breaking change**] Deprecated the following endpoints: - - *getConnection* + - *getConnection* -> use `/sensors` (GET) instead - *getMeterData* -> use `/sensors/data` (GET) instead - *getPrognosis* -> use `/sensors/data` (GET) instead - *getService* diff --git a/flexmeasures/api/v3_0/implementations/sensors.py b/flexmeasures/api/v3_0/implementations/sensors.py index 3e173c2e5..256263dc7 100644 --- a/flexmeasures/api/v3_0/implementations/sensors.py +++ b/flexmeasures/api/v3_0/implementations/sensors.py @@ -1,15 +1,39 @@ import json from flask_classful import FlaskView, route +from flask_json import as_json from flask_security import auth_required from timely_beliefs import BeliefsDataFrame -from webargs.flaskparser import use_args +from webargs.flaskparser import use_args, use_kwargs from flexmeasures.api.common.schemas.sensor_data import ( GetSensorDataSchema, PostSensorDataSchema, ) +from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.utils.api_utils import save_and_enqueue +from flexmeasures.auth.decorators import permission_required_for_context +from flexmeasures.data.models.user import Account +from flexmeasures.data.schemas.sensors import SensorSchema +from flexmeasures.data.services.sensors import get_sensors + +sensors_schema = SensorSchema(many=True) + + +@use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + }, + location="query", +) +@permission_required_for_context("read", arg_name="account") +@as_json +def get(account: Account): + """List sensors of an account.""" + sensors = get_sensors(account_name=account.name) + return sensors_schema.dump(sensors), 200 class SensorAPI(FlaskView): @@ -18,6 +42,13 @@ class SensorAPI(FlaskView): trailing_slash = False decorators = [auth_required()] + def index(self): + """API endpoint to get sensors. + + .. :quickref: Sensor; Download sensor list + """ + return get() + @route("/data", methods=["POST"]) @use_args( PostSensorDataSchema(), diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 62b14d853..714808446 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -29,6 +29,7 @@ class Meta: unit = ma.auto_field(required=True) timezone = ma.auto_field() event_resolution = fields.TimeDelta(required=True, precision="minutes") + entity_address = fields.String(dump_only=True) @validates("unit") def validate_unit(self, unit: str): diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py new file mode 100644 index 000000000..1b09cf8b5 --- /dev/null +++ b/flexmeasures/data/services/sensors.py @@ -0,0 +1,26 @@ +from typing import Optional, List + +from werkzeug.exceptions import NotFound + +from flexmeasures import Sensor, Account +from flexmeasures.data.models.generic_assets import GenericAsset + + +def get_sensors( + account_name: Optional[str] = None, +) -> List[Sensor]: + """Return a list of Sensor objects. + + :param account_name: optionally, filter by account name. + """ + sensor_query = Sensor.query.join(GenericAsset).filter( + Sensor.generic_asset_id == GenericAsset.id + ) + + if account_name is not None: + account = Account.query.filter(Account.name == account_name).one_or_none() + if not account: + raise NotFound(f"There is no account named {account_name}!") + sensor_query = sensor_query.filter(GenericAsset.owner == account) + + return sensor_query.all() From c06d76840a523ea98be6e6d147e6a4ff74f3a034 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 17:23:28 +0100 Subject: [PATCH 74/99] Complete docstring Signed-off-by: F.N. Claessen --- .../api/v3_0/implementations/sensors.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/flexmeasures/api/v3_0/implementations/sensors.py b/flexmeasures/api/v3_0/implementations/sensors.py index 256263dc7..554606efb 100644 --- a/flexmeasures/api/v3_0/implementations/sensors.py +++ b/flexmeasures/api/v3_0/implementations/sensors.py @@ -46,6 +46,35 @@ def index(self): """API endpoint to get sensors. .. :quickref: Sensor; Download sensor list + + This endpoint returns all accessible sensors. + Accessible sensors are sensors in the same account as the current user. + Only admins can use this endpoint to fetch sensors from a different account (by using the `account_id` query parameter). + + **Example response** + + An example of one sensor being returned: + + .. sourcecode:: json + + [ + { + "entity_address": "ea1.2021-01.io.flexmeasures.company:fm1.42", + "event_resolution": 15, + "generic_asset_id": 1, + "name": "Gas demand", + "timezone": "Europe/Amsterdam", + "unit": "m\u00b3/h" + } + ] + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER """ return get() From 257b5b53f838e4e4ca97df77b53faa2404741d4b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 17:52:27 +0100 Subject: [PATCH 75/99] Add extra tips to API changelog on how to upgrade from v2 to v3 Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index e07eaf740..9dd45b366 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -15,12 +15,12 @@ v3.0-0 | 2022-03-15 - [**Breaking change**] Deprecated the following endpoints: - *getConnection* -> use `/sensors` (GET) instead - - *getMeterData* -> use `/sensors/data` (GET) instead - - *getPrognosis* -> use `/sensors/data` (GET) instead + - *getMeterData* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" + - *getPrognosis* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor" - *getService* - - *postMeterData* -> use `/sensors/data` (POST) instead - - *postPriceData* -> use `/sensors/data` (POST) instead - - *postPrognosis* -> use `/sensors/data` (POST) instead + - *postMeterData* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" + - *postPriceData* -> use `/sensors/data` (POST) instead, replacing the "market" field with "sensor" + - *postPrognosis* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor" - *postWeatherData* -> use `/sensors/data` (POST) instead - *restoreData* From b8899921e5205530762235e63b43d47645914c93 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 15 Mar 2022 17:56:27 +0100 Subject: [PATCH 76/99] Fix check for belief timing against message type Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 089523864..38f8a002e 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -156,9 +156,9 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict: horizons_at_least = sensor_data_description.get("horizon", None) horizons_at_most = None _type = sensor_data_description.get("type", None) - if _type == "PostMeterDataRequest": + if _type == "GetMeterDataRequest": horizons_at_most = timedelta(0) - elif _type == "PostPrognosisRequest": + elif _type == "GetPrognosisRequest": if horizons_at_least is None: horizons_at_least = timedelta(0) else: From 6037d606b02ed3a2cf849e8988f169a1cb936461 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 16 Mar 2022 17:33:00 +0100 Subject: [PATCH 77/99] No need to use implementations folder anymore Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/__init__.py | 4 ++-- flexmeasures/api/v3_0/implementations/__init__.py | 0 flexmeasures/api/v3_0/{implementations => }/sensors.py | 0 flexmeasures/api/v3_0/{implementations => }/users.py | 0 4 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 flexmeasures/api/v3_0/implementations/__init__.py rename flexmeasures/api/v3_0/{implementations => }/sensors.py (100%) rename flexmeasures/api/v3_0/{implementations => }/users.py (100%) diff --git a/flexmeasures/api/v3_0/__init__.py b/flexmeasures/api/v3_0/__init__.py index 51cf9b134..620ccf4fa 100644 --- a/flexmeasures/api/v3_0/__init__.py +++ b/flexmeasures/api/v3_0/__init__.py @@ -1,7 +1,7 @@ from flask import Flask -from flexmeasures.api.v3_0.implementations.sensors import SensorAPI -from flexmeasures.api.v3_0.implementations.users import UserAPI +from flexmeasures.api.v3_0.sensors import SensorAPI +from flexmeasures.api.v3_0.users import UserAPI def register_at(app: Flask): diff --git a/flexmeasures/api/v3_0/implementations/__init__.py b/flexmeasures/api/v3_0/implementations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/flexmeasures/api/v3_0/implementations/sensors.py b/flexmeasures/api/v3_0/sensors.py similarity index 100% rename from flexmeasures/api/v3_0/implementations/sensors.py rename to flexmeasures/api/v3_0/sensors.py diff --git a/flexmeasures/api/v3_0/implementations/users.py b/flexmeasures/api/v3_0/users.py similarity index 100% rename from flexmeasures/api/v3_0/implementations/users.py rename to flexmeasures/api/v3_0/users.py From 16130a16a66fd5a64dcafddcb274d7b72cfd8d89 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 16 Mar 2022 17:35:45 +0100 Subject: [PATCH 78/99] Contain endpoint logic within SensorAPI class Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 36 ++++++++++++++------------------ 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 554606efb..7bf4eff52 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -20,35 +20,30 @@ sensors_schema = SensorSchema(many=True) -@use_kwargs( - { - "account": AccountIdField( - data_key="account_id", load_default=AccountIdField.load_current - ), - }, - location="query", -) -@permission_required_for_context("read", arg_name="account") -@as_json -def get(account: Account): - """List sensors of an account.""" - sensors = get_sensors(account_name=account.name) - return sensors_schema.dump(sensors), 200 - - class SensorAPI(FlaskView): route_base = "/sensors" trailing_slash = False decorators = [auth_required()] - def index(self): - """API endpoint to get sensors. + @route("/", methods=["GET"]) + @use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + }, + location="query", + ) + @permission_required_for_context("read", arg_name="account") + @as_json + def index(self, account: Account): + """API endpoint to list all sensors of an account. .. :quickref: Sensor; Download sensor list This endpoint returns all accessible sensors. - Accessible sensors are sensors in the same account as the current user. + Accessible sensors are sensors in the same account as the current user. Only admins can use this endpoint to fetch sensors from a different account (by using the `account_id` query parameter). **Example response** @@ -76,7 +71,8 @@ def index(self): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return get() + sensors = get_sensors(account_name=account.name) + return sensors_schema.dump(sensors), 200 @route("/data", methods=["POST"]) @use_args( From 1b6394f6f20c819940602cc0fdceff298ad9d8a8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 16 Mar 2022 17:55:45 +0100 Subject: [PATCH 79/99] Contain endpoint logic within UserAPI class Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/users.py | 84 ++++++++++++++-------------------- 1 file changed, 34 insertions(+), 50 deletions(-) diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index 99a6c62ca..4491dce8e 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -28,31 +28,6 @@ users_schema = UserSchema(many=True) -@use_kwargs( - { - "account": AccountIdField( - data_key="account_id", load_default=AccountIdField.load_current - ), - "include_inactive": fields.Bool(load_default=False), - }, - location="query", -) -@permission_required_for_context("read", arg_name="account") -@as_json -def get(account: Account, include_inactive: bool = False): - """List users of an account.""" - users = get_users(account_name=account.name, only_active=not include_inactive) - return users_schema.dump(users), 200 - - -@use_kwargs({"user": UserIdField(data_key="id")}, location="path") -@permission_required_for_context("read", arg_name="user") -@as_json -def fetch_one(user_id: int, user: UserModel): - """Fetch a given user""" - return user_schema.dump(user), 200 - - @use_args(UserSchema(partial=True)) @use_kwargs({"db_user": UserIdField(data_key="id")}, location="path") @permission_required_for_context("update", arg_name="db_user") @@ -76,28 +51,24 @@ def patch(id: int, user_data: dict, db_user: UserModel): return user_schema.dump(db_user), 200 -@use_kwargs({"user": UserIdField(data_key="id")}, location="path") -@permission_required_for_context("update", arg_name="user") -@as_json -def reset_password(user_id: int, user: UserModel): - """ - Reset the user's current password, cookies and auth tokens. - Send a password reset link to the user. - """ - set_random_password(user) - remove_cookie_and_token_access(user) - send_reset_password_instructions(user) - - # commit only if sending instructions worked, as well - db.session.commit() - - class UserAPI(FlaskView): route_base = "/users" trailing_slash = False - def index(self): - """API endpoint to get users. + @route("", methods=["GET"]) + @use_kwargs( + { + "account": AccountIdField( + data_key="account_id", load_default=AccountIdField.load_current + ), + "include_inactive": fields.Bool(load_default=False), + }, + location="query", + ) + @permission_required_for_context("read", arg_name="account") + @as_json + def index(self, account: Account, include_inactive: bool = False): + """API endpoint to list all users of an account. .. :quickref: User; Download user list @@ -134,9 +105,14 @@ def index(self): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return get() - - def get(self, id: int): + users = get_users(account_name=account.name, only_active=not include_inactive) + return users_schema.dump(users), 200 + + @route("/") + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("read", arg_name="user") + @as_json + def get(self, id: int, user: UserModel): """API endpoint to get a user. .. :quickref: User; Get a user @@ -166,7 +142,7 @@ def get(self, id: int): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return fetch_one(id) + return user_schema.dump(user), 200 def patch(self, id: int): """API endpoint to patch user data. @@ -216,8 +192,11 @@ def patch(self, id: int): return patch(id) @route("//password-reset", methods=["PATCH"]) - def reset_user_password(self, id: int): - """API endpoint to reset the user password. They'll get an email to choose a new password. + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("update", arg_name="user") + @as_json + def reset_user_password(self, id: int, user: UserModel): + """API endpoint to reset the user's current password, cookies and auth tokens, and to email a password reset link to the user. .. :quickref: User; Password reset @@ -236,4 +215,9 @@ def reset_user_password(self, id: int): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return reset_password(id) + set_random_password(user) + remove_cookie_and_token_access(user) + send_reset_password_instructions(user) + + # commit only if sending instructions worked, as well + db.session.commit() From c16a3bf9b138977ca3a0b5a69b2ca85a5cfd0a33 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 15:50:46 +0100 Subject: [PATCH 80/99] Fix mock email and type annotations Signed-off-by: F.N. Claessen --- flexmeasures/ui/tests/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/tests/utils.py b/flexmeasures/ui/tests/utils.py index 1f2e0c747..9d2aa8953 100644 --- a/flexmeasures/ui/tests/utils.py +++ b/flexmeasures/ui/tests/utils.py @@ -1,4 +1,5 @@ import copy +from typing import List, Union from flask import url_for @@ -22,7 +23,7 @@ def mock_asset_response( account_id: int = 1, as_list: bool = True, multiple: bool = False, -) -> dict: +) -> Union[dict, List[dict]]: asset = dict( id=asset_id, name="TestAsset", @@ -48,7 +49,7 @@ def mock_user_response( active: bool = True, as_list: bool = True, multiple: bool = False, -) -> dict: +) -> Union[dict, List[dict]]: user = dict( id=user_id, username=username, @@ -64,7 +65,7 @@ def mock_user_response( user2 = copy.deepcopy(user) user2["id"] = 2 user2["username"] = "Bert" - user2["email"] = ("bert@seita.nl",) + user2["email"] = "bert@seita.nl" user_list.append(user2) return user_list return user From d4e99dbba599a16e15ccf5f0cf6abe6f9d9a5e14 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 15:57:45 +0100 Subject: [PATCH 81/99] Initialize schemas within SensorAPI and UserAPI class Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 4 +--- flexmeasures/api/v3_0/users.py | 9 +++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 7bf4eff52..b0926cf2a 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -17,8 +17,6 @@ from flexmeasures.data.schemas.sensors import SensorSchema from flexmeasures.data.services.sensors import get_sensors -sensors_schema = SensorSchema(many=True) - class SensorAPI(FlaskView): @@ -72,7 +70,7 @@ def index(self, account: Account): :status 403: INVALID_SENDER """ sensors = get_sensors(account_name=account.name) - return sensors_schema.dump(sensors), 200 + return SensorSchema(many=True).dump(sensors), 200 @route("/data", methods=["POST"]) @use_args( diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index 4491dce8e..b3b370600 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -24,9 +24,6 @@ Both POST (to create) and DELETE are not accessible via the API, but as CLI functions. """ -user_schema = UserSchema() -users_schema = UserSchema(many=True) - @use_args(UserSchema(partial=True)) @use_kwargs({"db_user": UserIdField(data_key="id")}, location="path") @@ -48,7 +45,7 @@ def patch(id: int, user_data: dict, db_user: UserModel): db.session.commit() except IntegrityError as ie: return dict(message="Duplicate user already exists", detail=ie._message()), 400 - return user_schema.dump(db_user), 200 + return UserSchema().dump(db_user), 200 class UserAPI(FlaskView): @@ -106,7 +103,7 @@ def index(self, account: Account, include_inactive: bool = False): :status 403: INVALID_SENDER """ users = get_users(account_name=account.name, only_active=not include_inactive) - return users_schema.dump(users), 200 + return UserSchema(many=True).dump(users), 200 @route("/") @use_kwargs({"user": UserIdField(data_key="id")}, location="path") @@ -142,7 +139,7 @@ def get(self, id: int, user: UserModel): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return user_schema.dump(user), 200 + return UserSchema().dump(user), 200 def patch(self, id: int): """API endpoint to patch user data. From cdb7704e18bf63cf3ee79525b8a6d9fc68967338 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 16:16:51 +0100 Subject: [PATCH 82/99] Typos Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/users.py b/flexmeasures/api/common/schemas/users.py index 84e14c943..b704ae2ef 100644 --- a/flexmeasures/api/common/schemas/users.py +++ b/flexmeasures/api/common/schemas/users.py @@ -7,7 +7,7 @@ class AccountIdField(fields.Integer): """ - Field that represents an account ID. It de-serializes from the account id to an account instance. + Field that represents an account ID. It deserializes from the account id to an account instance. """ def _deserialize(self, account_id: str, attr, obj, **kwargs) -> Account: @@ -30,7 +30,7 @@ def load_current(cls): class UserIdField(fields.Integer): """ - Field that represents a user ID. It de-serializes from the user id to a user instance. + Field that represents a user ID. It deserializes from the user id to a user instance. """ def __init__(self, *args, **kwargs): From 4c982bbd1cc222a6e54f25f18c31034ac3c99a38 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 16:17:35 +0100 Subject: [PATCH 83/99] Fix 404 error message Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/users.py b/flexmeasures/api/common/schemas/users.py index b704ae2ef..4470e9968 100644 --- a/flexmeasures/api/common/schemas/users.py +++ b/flexmeasures/api/common/schemas/users.py @@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs): def _deserialize(self, user_id: int, attr, obj, **kwargs) -> User: user: User = User.query.filter_by(id=int(user_id)).one_or_none() if user is None: - raise abort(404, f"User {id} not found") + raise abort(404, f"User {user_id} not found") return user def _serialize(self, user: User, attr, data, **kwargs) -> int: From ea1e1b6605e40d1ba83d009a773070584656878e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 16:38:13 +0100 Subject: [PATCH 84/99] Remove unused variable declaration Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/assets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index ee68f6d80..0e03c6fd8 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -344,7 +344,6 @@ def delete_with_data(self, id: str): def _set_account(asset_form: NewAssetForm) -> Tuple[Optional[Account], Optional[str]]: """Set an account for the to-be-created asset. Return the account (if available) and an error message""" - account = None account_error = None if asset_form.account_id.data == -1: From 38d3bc15cf70338bdabcef469f23e45a65e5f9ce Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 16:48:06 +0100 Subject: [PATCH 85/99] Clarify use of headers in test Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_api_v3_0_users.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 1b7314f3d..43378048f 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -105,10 +105,13 @@ def test_edit_user(client): assert user_edit_response.status_code == 401 # admin can deactivate user2, other changes will be ignored # (id is in the User schema of the API, but we ignore it) - headers = {"content-type": "application/json", "Authorization": admin_auth_token} + admin_headers = { + "content-type": "application/json", + "Authorization": admin_auth_token, + } user_edit_response = client.patch( url_for("UserAPI:patch", id=user2_id), - headers=headers, + headers=admin_headers, json={"active": False, "id": 888}, ) print("Server responded with:\n%s" % user_edit_response.json) @@ -118,10 +121,9 @@ def test_edit_user(client): assert user2.active is False assert user2.id == user2_id # admin can edit themselves but not sensitive fields - headers = {"content-type": "application/json", "Authorization": admin_auth_token} user_edit_response = client.patch( url_for("UserAPI:patch", id=admin_id), - headers=headers, + headers=admin_headers, json={"active": False}, ) print("Server responded with:\n%s" % user_edit_response.json) From 9d98f71674344be520452cfecd16d5ba049f835e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 17 Mar 2022 17:00:28 +0100 Subject: [PATCH 86/99] Concerning patching user ids and user account ids, set dump_only for these fields instead of ignoring them Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_api_v3_0_users.py | 5 ++--- flexmeasures/api/v3_0/users.py | 4 ++-- flexmeasures/data/schemas/users.py | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 43378048f..83c38a28f 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -103,8 +103,7 @@ def test_edit_user(client): json={}, ) assert user_edit_response.status_code == 401 - # admin can deactivate user2, other changes will be ignored - # (id is in the User schema of the API, but we ignore it) + # admin can deactivate user2 admin_headers = { "content-type": "application/json", "Authorization": admin_auth_token, @@ -112,7 +111,7 @@ def test_edit_user(client): user_edit_response = client.patch( url_for("UserAPI:patch", id=user2_id), headers=admin_headers, - json={"active": False, "id": 888}, + json={"active": False}, ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 200 diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index b3b370600..35eaa53f3 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -151,7 +151,7 @@ def patch(self, id: int): Only the user themselves or admins are allowed to update its data, while a non-admin can only edit a few of their own fields. - Several fields are not allowed to be updated, e.g. id and account_id. They are ignored. + Several fields are not allowed to be updated, e.g. id and account_id. **Example request** @@ -163,7 +163,7 @@ def patch(self, id: int): **Example response** - The whole user is returned in the response: + The following user fields are returned: .. sourcecode:: json diff --git a/flexmeasures/data/schemas/users.py b/flexmeasures/data/schemas/users.py index dbec66c28..f1c175c95 100644 --- a/flexmeasures/data/schemas/users.py +++ b/flexmeasures/data/schemas/users.py @@ -19,10 +19,10 @@ def validate_timezone(self, timezone): if timezone not in all_timezones: raise ValidationError(f"Timezone {timezone} doesn't exist.") - id = ma.auto_field() + id = ma.auto_field(dump_only=True) email = ma.auto_field(required=True, validate=validate.Email) username = ma.auto_field(required=True) - account_id = ma.auto_field() + account_id = ma.auto_field(dump_only=True) active = ma.auto_field() timezone = ma.auto_field() flexmeasures_roles = ma.auto_field() From ca3154ff5aa28be444c56759d5f75b6b28b39384 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 18 Mar 2022 13:57:57 +0100 Subject: [PATCH 87/99] Add test cases for unexpected fields Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_api_v3_0_users.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py index 83c38a28f..0ae15b7a1 100644 --- a/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py +++ b/flexmeasures/api/v3_0/tests/test_api_v3_0_users.py @@ -129,7 +129,15 @@ def test_edit_user(client): assert user_edit_response.status_code == 403 -def test_edit_user_with_unexpected_fields(client): +@pytest.mark.parametrize( + "unexpected_fields", + [ + dict(password="I-should-not-be-sending-this"), # not part of the schema + dict(id=10), # id is a dump_only field + dict(account_id=10), # account_id is a dump_only field + ], +) +def test_edit_user_with_unexpected_fields(client, unexpected_fields: dict): """Sending unexpected fields (not in Schema) is an Unprocessable Entity error.""" with UserContext("test_prosumer_user_2@seita.nl") as user2: user2_id = user2.id @@ -141,7 +149,7 @@ def test_edit_user_with_unexpected_fields(client): "content-type": "application/json", "Authorization": admin_auth_token, }, - json={"active": False, "password": "I-should-not-be-sending-this"}, + json={**{"active": False}, **unexpected_fields}, ) print("Server responded with:\n%s" % user_edit_response.json) assert user_edit_response.status_code == 422 From 71add340ee266d455fc331336ec64ef4b257a32a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 18 Mar 2022 14:16:48 +0100 Subject: [PATCH 88/99] Also move patch endpoint logic into UserAPI class; this one was a bit more tricky, because the 'id' fields was being passed to the function twice somehow; use_kwargs instead of use_args was needed to avoid this Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/users.py | 57 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index 35eaa53f3..a821e3f83 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -1,7 +1,7 @@ from flask_classful import FlaskView, route from marshmallow import fields from sqlalchemy.exc import IntegrityError -from webargs.flaskparser import use_kwargs, use_args +from webargs.flaskparser import use_kwargs from flask_security import current_user from flask_security.recoverable import send_reset_password_instructions from flask_json import as_json @@ -25,29 +25,6 @@ """ -@use_args(UserSchema(partial=True)) -@use_kwargs({"db_user": UserIdField(data_key="id")}, location="path") -@permission_required_for_context("update", arg_name="db_user") -@as_json -def patch(id: int, user_data: dict, db_user: UserModel): - """Update a user given its identifier""" - allowed_fields = ["email", "username", "active", "timezone", "flexmeasures_roles"] - for k, v in [(k, v) for k, v in user_data.items() if k in allowed_fields]: - if current_user.id == db_user.id and k in ("active", "flexmeasures_roles"): - raise Forbidden( - "Users who edit themselves cannot edit security-sensitive fields." - ) - setattr(db_user, k, v) - if k == "active" and v is False: - remove_cookie_and_token_access(db_user) - db.session.add(db_user) - try: - db.session.commit() - except IntegrityError as ie: - return dict(message="Duplicate user already exists", detail=ie._message()), 400 - return UserSchema().dump(db_user), 200 - - class UserAPI(FlaskView): route_base = "/users" trailing_slash = False @@ -141,7 +118,12 @@ def get(self, id: int, user: UserModel): """ return UserSchema().dump(user), 200 - def patch(self, id: int): + @route("/", methods=["PATCH"]) + @use_kwargs(UserSchema(partial=True)) + @use_kwargs({"user": UserIdField(data_key="id")}, location="path") + @permission_required_for_context("update", arg_name="user") + @as_json + def patch(self, id: int, user: UserModel, **user_data): """API endpoint to patch user data. .. :quickref: User; Patch data for an existing user @@ -186,7 +168,30 @@ def patch(self, id: int): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - return patch(id) + allowed_fields = [ + "email", + "username", + "active", + "timezone", + "flexmeasures_roles", + ] + for k, v in [(k, v) for k, v in user_data.items() if k in allowed_fields]: + if current_user.id == user.id and k in ("active", "flexmeasures_roles"): + raise Forbidden( + "Users who edit themselves cannot edit security-sensitive fields." + ) + setattr(user, k, v) + if k == "active" and v is False: + remove_cookie_and_token_access(user) + db.session.add(user) + try: + db.session.commit() + except IntegrityError as ie: + return ( + dict(message="Duplicate user already exists", detail=ie._message()), + 400, + ) + return UserSchema().dump(user), 200 @route("//password-reset", methods=["PATCH"]) @use_kwargs({"user": UserIdField(data_key="id")}, location="path") From 1f07045f8c9cea2ccf9e544e5cf596cb368e35ad Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 18 Mar 2022 14:46:44 +0100 Subject: [PATCH 89/99] Move test utils from dev to v3_0 Signed-off-by: F.N. Claessen --- .../api/dev/tests/test_sensor_data.py | 2 +- .../dev/tests/test_sensor_data_fresh_db.py | 2 +- flexmeasures/api/dev/tests/utils.py | 25 ------------------- flexmeasures/api/v3_0/tests/utils.py | 23 +++++++++++++++++ 4 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 flexmeasures/api/v3_0/tests/utils.py diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/dev/tests/test_sensor_data.py index 5e12b899c..81180dba7 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data.py +++ b/flexmeasures/api/dev/tests/test_sensor_data.py @@ -2,7 +2,7 @@ import pytest from flexmeasures.api.tests.utils import get_auth_token -from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor +from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor @pytest.mark.parametrize("use_auth", [False, True]) diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py index 56d2af2be..d55923d8d 100644 --- a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py @@ -2,7 +2,7 @@ from flask import url_for from flexmeasures.api.tests.utils import get_auth_token -from flexmeasures.api.dev.tests.utils import make_sensor_data_request_for_gas_sensor +from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor from flexmeasures.data.models.time_series import TimedBelief, Sensor diff --git a/flexmeasures/api/dev/tests/utils.py b/flexmeasures/api/dev/tests/utils.py index 70ae69749..aecd33834 100644 --- a/flexmeasures/api/dev/tests/utils.py +++ b/flexmeasures/api/dev/tests/utils.py @@ -1,28 +1,3 @@ -from flexmeasures.data.models.time_series import Sensor - - -def make_sensor_data_request_for_gas_sensor( - num_values: int = 6, duration: str = "PT1H", unit: str = "m³" -) -> dict: - """Creates request to post sensor data for a gas sensor. - This particular gas sensor measures units of m³/h with a 10-minute resolution. - """ - sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() - message: dict = { - "type": "PostSensorDataRequest", - "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", - "values": num_values * [-11.28], - "start": "2021-06-07T00:00:00+02:00", - "duration": duration, - "horizon": "PT0H", - "unit": unit, - } - if num_values == 1: - # flatten [] to - message["values"] = message["values"][0] - return message - - def get_asset_post_data(account_id: int = 1, asset_type_id: int = 1) -> dict: post_data = { "name": "Test battery 2", diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py new file mode 100644 index 000000000..83697ba59 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -0,0 +1,23 @@ +from flexmeasures import Sensor + + +def make_sensor_data_request_for_gas_sensor( + num_values: int = 6, duration: str = "PT1H", unit: str = "m³" +) -> dict: + """Creates request to post sensor data for a gas sensor. + This particular gas sensor measures units of m³/h with a 10-minute resolution. + """ + sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() + message: dict = { + "type": "PostSensorDataRequest", + "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", + "values": num_values * [-11.28], + "start": "2021-06-07T00:00:00+02:00", + "duration": duration, + "horizon": "PT0H", + "unit": unit, + } + if num_values == 1: + # flatten [] to + message["values"] = message["values"][0] + return message \ No newline at end of file From d8b27909d71fd23a1a75e4612e649976d654663d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 18 Mar 2022 23:36:52 +0100 Subject: [PATCH 90/99] Finish moving sensor data tests from dev to v3_0 Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/tests/__init__.py | 0 flexmeasures/api/dev/tests/conftest.py | 32 -------------- flexmeasures/api/v3_0/tests/conftest.py | 44 ++++++++++++++++++- .../{dev => v3_0}/tests/test_sensor_data.py | 0 .../tests/test_sensor_data_fresh_db.py | 0 flexmeasures/api/v3_0/tests/utils.py | 2 +- 6 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 flexmeasures/api/dev/tests/__init__.py rename flexmeasures/api/{dev => v3_0}/tests/test_sensor_data.py (100%) rename flexmeasures/api/{dev => v3_0}/tests/test_sensor_data_fresh_db.py (100%) diff --git a/flexmeasures/api/dev/tests/__init__.py b/flexmeasures/api/dev/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index df97ec761..356c861ff 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -1,10 +1,5 @@ -from datetime import timedelta - import pytest -from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset -from flexmeasures.data.models.time_series import Sensor - @pytest.fixture(scope="module") def setup_api_test_data(db, setup_roles_users, setup_generic_assets): @@ -12,7 +7,6 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) - add_gas_sensor(db, setup_roles_users["Test Supplier User"]) @pytest.fixture(scope="function") @@ -23,29 +17,3 @@ def setup_api_fresh_test_data( Set up fresh data for API dev tests. """ print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) - for sensor in Sensor.query.all(): - fresh_db.delete(sensor) - add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) - - -def add_gas_sensor(db, test_supplier_user): - incineration_type = GenericAssetType( - name="waste incinerator", - ) - db.session.add(incineration_type) - db.session.flush() - incineration_asset = GenericAsset( - name="incineration line", - generic_asset_type=incineration_type, - account_id=test_supplier_user.account_id, - ) - db.session.add(incineration_asset) - db.session.flush() - gas_sensor = Sensor( - name="some gas sensor", - unit="m³/h", - event_resolution=timedelta(minutes=10), - generic_asset=incineration_asset, - ) - db.session.add(gas_sensor) - gas_sensor.owner = test_supplier_user.account diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index be983fd5a..75ab132a7 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -1,13 +1,32 @@ +from datetime import timedelta + import pytest from flask_security import SQLAlchemySessionUserDatastore, hash_password +from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset +from flexmeasures.data.models.time_series import Sensor + @pytest.fixture(scope="module", autouse=True) -def setup_api_test_data(db, setup_roles_users): +def setup_api_test_data(db, setup_roles_users, setup_generic_assets): """ Set up data for API v3.0 tests. """ print("Setting up data for API v3.0 tests on %s" % db.engine) + add_gas_sensor(db, setup_roles_users["Test Supplier User"]) + + +@pytest.fixture(scope="function") +def setup_api_fresh_test_data( + fresh_db, setup_roles_users_fresh_db, setup_generic_assets_fresh_db +): + """ + Set up fresh data for API dev tests. + """ + print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) + for sensor in Sensor.query.all(): + fresh_db.delete(sensor) + add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) @pytest.fixture(scope="module") @@ -25,3 +44,26 @@ def setup_inactive_user(db, setup_accounts, setup_roles_users): account_id=setup_accounts["Prosumer"].id, active=False, ) + + +def add_gas_sensor(db, test_supplier_user): + incineration_type = GenericAssetType( + name="waste incinerator", + ) + db.session.add(incineration_type) + db.session.flush() + incineration_asset = GenericAsset( + name="incineration line", + generic_asset_type=incineration_type, + account_id=test_supplier_user.account_id, + ) + db.session.add(incineration_asset) + db.session.flush() + gas_sensor = Sensor( + name="some gas sensor", + unit="m³/h", + event_resolution=timedelta(minutes=10), + generic_asset=incineration_asset, + ) + db.session.add(gas_sensor) + gas_sensor.owner = test_supplier_user.account diff --git a/flexmeasures/api/dev/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py similarity index 100% rename from flexmeasures/api/dev/tests/test_sensor_data.py rename to flexmeasures/api/v3_0/tests/test_sensor_data.py diff --git a/flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py similarity index 100% rename from flexmeasures/api/dev/tests/test_sensor_data_fresh_db.py rename to flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py index 83697ba59..c7477dd91 100644 --- a/flexmeasures/api/v3_0/tests/utils.py +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -20,4 +20,4 @@ def make_sensor_data_request_for_gas_sensor( if num_values == 1: # flatten [] to message["values"] = message["values"][0] - return message \ No newline at end of file + return message From 0589009812d23e4ea2351ddf6615648cd9221f69 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 18 Mar 2022 23:47:00 +0100 Subject: [PATCH 91/99] Fix asset tests in dev API by moving back some conftest logic Signed-off-by: F.N. Claessen --- flexmeasures/api/dev/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 356c861ff..6887661e2 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -1,5 +1,8 @@ import pytest +from flexmeasures.api.v3_0.tests.conftest import add_gas_sensor +from flexmeasures.data.models.time_series import Sensor + @pytest.fixture(scope="module") def setup_api_test_data(db, setup_roles_users, setup_generic_assets): @@ -7,6 +10,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) + add_gas_sensor(db, setup_roles_users["Test Supplier User"]) @pytest.fixture(scope="function") @@ -17,3 +21,6 @@ def setup_api_fresh_test_data( Set up fresh data for API dev tests. """ print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) + for sensor in Sensor.query.all(): + fresh_db.delete(sensor) + add_gas_sensor(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) From 3743032ae887ed296bb4fd3a0108dd1d43ab828f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:01:06 +0100 Subject: [PATCH 92/99] Instantiate schemas outside of endpoint logic to minimize response time Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 11 ++++++++--- flexmeasures/api/v3_0/users.py | 13 +++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index b0926cf2a..4f625b5a5 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -17,6 +17,11 @@ from flexmeasures.data.schemas.sensors import SensorSchema from flexmeasures.data.services.sensors import get_sensors +# Instantiate schemas outside of endpoint logic to minimize response time +get_sensor_schema = GetSensorDataSchema() +post_sensor_schema = PostSensorDataSchema() +sensors_schema = SensorSchema(many=True) + class SensorAPI(FlaskView): @@ -70,11 +75,11 @@ def index(self, account: Account): :status 403: INVALID_SENDER """ sensors = get_sensors(account_name=account.name) - return SensorSchema(many=True).dump(sensors), 200 + return sensors_schema.dump(sensors), 200 @route("/data", methods=["POST"]) @use_args( - PostSensorDataSchema(), + post_sensor_schema, location="json", ) def post_data(self, bdf: BeliefsDataFrame): @@ -108,7 +113,7 @@ def post_data(self, bdf: BeliefsDataFrame): @route("/data", methods=["GET"]) @use_args( - GetSensorDataSchema(), + get_sensor_schema, location="query", ) def get_data(self, response: dict): diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index a821e3f83..39db098fb 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -24,6 +24,11 @@ Both POST (to create) and DELETE are not accessible via the API, but as CLI functions. """ +# Instantiate schemas outside of endpoint logic to minimize response time +user_schema = UserSchema() +users_schema = UserSchema(many=True) +partial_user_schema = UserSchema(partial=True) + class UserAPI(FlaskView): route_base = "/users" @@ -80,7 +85,7 @@ def index(self, account: Account, include_inactive: bool = False): :status 403: INVALID_SENDER """ users = get_users(account_name=account.name, only_active=not include_inactive) - return UserSchema(many=True).dump(users), 200 + return users_schema.dump(users), 200 @route("/") @use_kwargs({"user": UserIdField(data_key="id")}, location="path") @@ -116,10 +121,10 @@ def get(self, id: int, user: UserModel): :status 401: UNAUTHORIZED :status 403: INVALID_SENDER """ - return UserSchema().dump(user), 200 + return user_schema.dump(user), 200 @route("/", methods=["PATCH"]) - @use_kwargs(UserSchema(partial=True)) + @use_kwargs(partial_user_schema) @use_kwargs({"user": UserIdField(data_key="id")}, location="path") @permission_required_for_context("update", arg_name="user") @as_json @@ -191,7 +196,7 @@ def patch(self, id: int, user: UserModel, **user_data): dict(message="Duplicate user already exists", detail=ie._message()), 400, ) - return UserSchema().dump(user), 200 + return user_schema.dump(user), 200 @route("//password-reset", methods=["PATCH"]) @use_kwargs({"user": UserIdField(data_key="id")}, location="path") From bf10bf35300d93dc7a29f1c2d01c09647d3e8c3c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:04:21 +0100 Subject: [PATCH 93/99] Add mentions of http status to docstrings Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 19 +++++++++++++++++++ flexmeasures/api/v3_0/users.py | 3 +++ 2 files changed, 22 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 4f625b5a5..f327ce95e 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -73,6 +73,7 @@ def index(self, account: Account): :status 400: INVALID_REQUEST :status 401: UNAUTHORIZED :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ sensors = get_sensors(account_name=account.name) return sensors_schema.dump(sensors), 200 @@ -107,6 +108,15 @@ def post_data(self, bdf: BeliefsDataFrame): The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but FlexMeasures will attempt to upsample lower resolutions. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ response, code = save_and_enqueue(bdf) return response, code @@ -133,5 +143,14 @@ def get_data(self, response: dict): } The unit has to be convertible from the sensor's unit. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ return json.dumps(response) diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index 39db098fb..c425ea7da 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -83,6 +83,7 @@ def index(self, account: Account, include_inactive: bool = False): :status 400: INVALID_REQUEST :status 401: UNAUTHORIZED :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ users = get_users(account_name=account.name, only_active=not include_inactive) return users_schema.dump(users), 200 @@ -120,6 +121,7 @@ def get(self, id: int, user: UserModel): :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS :status 401: UNAUTHORIZED :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ return user_schema.dump(user), 200 @@ -221,6 +223,7 @@ def reset_user_password(self, id: int, user: UserModel): :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS :status 401: UNAUTHORIZED :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY """ set_random_password(user) remove_cookie_and_token_access(user) From f1590946d5d42d8a20e02fe67edfc0424426654e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:10:57 +0100 Subject: [PATCH 94/99] Undocument getService from the API introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 34 ++---------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 414b58eee..5eb0f75ac 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -41,7 +41,7 @@ Let's see what the ``/api`` endpoint returns: >>> res = requests.get("https://company.flexmeasures.io/api") >>> res.json() {'flexmeasures_version': '0.9.0', - 'message': 'For these API versions a public endpoint is available, listing its service. For example: /api/v2_0/getService and /api/v3_0/getService. An authentication token can be requested at: /api/requestAuthToken', + 'message': 'For these API versions endpoints are available. An authentication token can be requested at: /api/requestAuthToken. For a list of services, see https://flexmeasures.readthedocs.io', 'status': 200, 'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0', 'v3_0'] } @@ -53,37 +53,7 @@ So this tells us which API versions exist. For instance, we know that the latest https://company.flexmeasures.io/api/v3_0 -Also, we can see that a list of endpoints which are available at (a version of) the FlexMeasures web service can be obtained by sending a ``getService`` request. An optional field "access" can be used to specify a user role for which to obtain only the relevant services. - -**Example request** - -Let's ask which endpoints are available for meter data companies (MDC): - -.. code-block:: html - - https://company.flexmeasures.io/api/v2_0/getService?access=MDC - - -**Example response** - -.. code-block:: json - - { - "type": "GetServiceResponse", - "version": "1.0", - "services": [ - { - "name": "getMeterData", - "access": ["Aggregator", "Supplier", "MDC", "DSO", "Prosumer", "ESCo"], - "description": "Request meter reading" - }, - { - "name": "postMeterData", - "access": ["MDC"], - "description": "Send meter reading" - } - ] - } +Also, we can see that a list of endpoints is available on https://flexmeasures.readthedocs.io for each of these versions. .. _api_auth: From 8572b8ff0cf03ff9ef168f2d76edc60aa7523788 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:23:39 +0100 Subject: [PATCH 95/99] Undocument USEF roles in introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 5eb0f75ac..08a974a0e 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -101,28 +101,13 @@ which gives a response like this if the credentials are correct: .. note:: Each access token has a limited lifetime, see :ref:`auth`. -Roles ------ - -We distinguish the following roles with different access rights to the individual services. Capitalised roles are defined by USEF: - -- public -- user -- admin -- Aggregator -- Supplier: an energy retailer (see :ref:`supplier`) -- Prosumer: owner of a grid connection (see :ref:`prosumer`) -- ESCo: an energy service company (see :ref:`esco`) -- MDC: a meter data company (see :ref:`mdc`) -- DSO: a distribution system operator (see :ref:`dso`) - .. _sources: Sources ------- Requests for data may limit the data selection by specifying a source, for example, a specific user. -USEF roles are also valid source selectors. +Account roles are also valid source selectors. For example, to obtain data originating from either a meter data company or user 42, include the following: .. code-block:: json @@ -131,6 +116,8 @@ For example, to obtain data originating from either a meter data company or user "sources": ["MDC", "42"], } +Here, "MDC" is the name of the account role for meter data companies. + Notation -------- All requests and responses to and from the web service should be valid JSON messages. From 1c013e980a294e680f82a39698abb4477209febe Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:25:44 +0100 Subject: [PATCH 96/99] Use "sensor" instead of "connection" in introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 08a974a0e..8767d1409 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -136,27 +136,26 @@ Throughout this document, keys are written in singular if a single value is list The API, however, does not distinguish between singular and plural key notation. -Connections and entity addresses +Sensors and entity addresses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A connection represents an end point of the grid, at which an electricity sensor (power meter) is located. -Connections should be identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], -which is mostly taken from IETF RFC 3720 [2]: +All sensors are identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], +which is mostly taken from IETF RFC 3720 [2]. This is the complete structure of an EA1 address: .. code-block:: json { - "connection": "ea1.{date code}.{reversed domain name}:{locally unique string}" + "sensor": "ea1.{date code}.{reversed domain name}:{locally unique string}" } -Here is a full example for a FlexMeasures connection address: +Here is a full example for an entity address of a sensor in FlexMeasures: .. code-block:: json { - "connection": "ea1.2021-02.io.flexmeasures.company:fm1.73" + "sensor": "ea1.2021-02.io.flexmeasures.company:fm1.73" } where FlexMeasures runs at `company.flexmeasures.io` (which the current domain owner started using in February 2021), and the locally unique string uses the `fm1` scheme (see below) to identify sensor ID 73. @@ -211,7 +210,7 @@ It uses the fact that all FlexMeasures sensors have unique IDs. .. todo:: UDI events are not yet modelled in the fm1 scheme The ``fm0`` scheme is the original scheme. -It identified different types of sensors (such as connections, weather sensors and markets) in different ways. +It identified different types of sensors (such as grid connections, weather sensors and markets) in different ways. The ``fm0`` scheme has been deprecated for the most part and is no longer supported officially. Only UDI events still need to be sent using the fm0 scheme. From dd7c59e7a4b8a7583593169bdae4aae891a097fe Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:27:29 +0100 Subject: [PATCH 97/99] Undocument group notation in introduction Signed-off-by: F.N. Claessen --- documentation/api/introduction.rst | 63 ------------------------------ 1 file changed, 63 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 8767d1409..43a9d9855 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -220,69 +220,6 @@ Only UDI events still need to be sent using the fm0 scheme. ea1.2021-01.io.flexmeasures:fm0.::: -Groups -^^^^^^ - -Data such as measurements, load prognoses and tariffs are usually stated per group of connections. -When the attributes "start", "duration" and "unit" are stated outside of "groups" they are inherited by each of the individual groups. For example: - -.. code-block:: json - - { - "groups": [ - { - "connections": [ - "ea1.2021-02.io.flexmeasures.company:fm1.71", - "ea1.2021-02.io.flexmeasures.company:fm1.72" - ], - "values": [ - 306.66, - 306.66, - 0, - 0, - 306.66, - 306.66 - ] - }, - { - "connection": "ea1.2021-02.io.flexmeasures.company:fm1.73" - "values": [ - 306.66, - 0, - 0, - 0, - 306.66, - 306.66 - ] - } - ], - "start": "2016-05-01T12:45:00Z", - "duration": "PT1H30M", - "unit": "MW" - } - -In case of a single group of connections, the message may be flattened to: - -.. code-block:: json - - { - "connections": [ - "ea1.2021-02.io.flexmeasures.company:fm1.71", - "ea1.2021-02.io.flexmeasures.company:fm1.72" - ], - "values": [ - 306.66, - 306.66, - 0, - 0, - 306.66, - 306.66 - ], - "start": "2016-05-01T12:45:00Z", - "duration": "PT1H30M", - "unit": "MW" - } - Timeseries ^^^^^^^^^^ From c808fe3c387c4d1f86897681c51d773a431ccbdd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:33:48 +0100 Subject: [PATCH 98/99] Update API changelog for rewrite of introduction Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 9dd45b366..cb05e42a2 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -24,6 +24,13 @@ v3.0-0 | 2022-03-15 - *postWeatherData* -> use `/sensors/data` (POST) instead - *restoreData* +- Changed the Introduction section: + + - Rewrote the section on service listing for API versions to refer to the public documentation. + - Rewrote the section on entity addresses to refer to *sensors* instead of *connections*. + - Rewrote the sections on roles and sources into a combined section that refers to account roles rather than USEF roles. + - Deprecated the section on group notation. + v2.0-4 | 2022-01-04 """"""""""""""""""" From 0358dfa30bb7761f7788a338f3e524742bc47af8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 22 Mar 2022 09:36:50 +0100 Subject: [PATCH 99/99] Update API changelog entry date Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index cb05e42a2..a35812ddb 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,7 +6,7 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. -v3.0-0 | 2022-03-15 +v3.0-0 | 2022-03-22 """"""""""""""""""" - Added REST endpoint for listing sensors: `/sensors` (GET).