From fa465c888237a58462de7c758ddf5bde26dde03f Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 5 Jul 2023 14:47:51 +0200 Subject: [PATCH 01/24] feat(sensors): adds fetch_one sensor endpoint to API Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 37 ++++++++++++++++++ .../api/v3_0/tests/test_sensors_api.py | 39 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 flexmeasures/api/v3_0/tests/test_sensors_api.py diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 9f61a67ed..ff110203e 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -46,6 +46,7 @@ get_sensor_schema = GetSensorDataSchema() post_sensor_schema = PostSensorDataSchema() sensors_schema = SensorSchema(many=True) +sensor_schema = SensorSchema() class SensorAPI(FlaskView): @@ -494,3 +495,39 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg d, s = request_processed() return dict(**response, **d), s + + @route("/", methods=["GET"]) + @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path") + @permission_required_for_context("read", arg_name="sensor") + @as_json + def fetch_one(self, id, sensor): + """Fetch a given sensor. + + .. :quickref: Sensor; Get a sensor + + This endpoint gets a sensor. + + **Example response** + + .. sourcecode:: json + + { + "name": "some gas sensor", + "unit": "m³/h", + "entity_address": "ea1.2023-08.localhost:fm1.1", + "event_resolution": 10, + "generic_asset_id": 4, + "timezone": "UTC", + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + sensor = Sensor.query.filter(Sensor.id == 1).one_or_none() + return sensor_schema.dump(sensor), 200 diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py new file mode 100644 index 000000000..68c836542 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -0,0 +1,39 @@ +from __future__ import annotations + + +from flask import url_for + + +from flexmeasures import Sensor +from flexmeasures.api.tests.utils import get_auth_token + + +def test_fetch_one_sensor( + client, + setup_api_test_data: dict[str, Sensor], +): + sensor_id = 1 + assert_response = { + "name": "some gas sensor", + "unit": "m³/h", + "entity_address": "ea1.2023-08.localhost:fm1.1", + "event_resolution": 10, + "generic_asset_id": 4, + "timezone": "UTC", + "status": 200, + } + headers = make_headers_for("test_supplier_user_4@seita.nl", client) + response = client.get( + url_for("SensorAPI:fetch_one", id=sensor_id), + headers=headers, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + assert response.json == assert_response + + +def make_headers_for(user_email: str | None, client) -> dict: + headers = {"content-type": "application/json"} + if user_email: + headers["Authorization"] = get_auth_token(client, user_email, "testtest") + return headers From 415f86141575870451324ab8596f85891bec44af Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 5 Jul 2023 14:57:49 +0200 Subject: [PATCH 02/24] feat(sensors): adds post sensor to API Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 56 +++++++++++++++++++ .../api/v3_0/tests/test_sensors_api.py | 47 ++++++++++++++++ flexmeasures/api/v3_0/tests/utils.py | 10 ++++ 3 files changed, 113 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index ff110203e..6de2b480c 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -531,3 +531,59 @@ def fetch_one(self, id, sensor): """ sensor = Sensor.query.filter(Sensor.id == 1).one_or_none() return sensor_schema.dump(sensor), 200 + + @route("", methods=["POST"]) + @permission_required_for_context( + "create-children", arg_loader=AccountIdField.load_current + ) + @use_args(sensor_schema) + def post(self, sensor_data: dict): + """Create new asset. + + .. :quickref: Asset; Create a new asset + + This endpoint creates a new asset. + + **Example request** + + .. sourcecode:: json + + { + "name": "Test battery", + "generic_asset_type_id": 2, + "account_id": 2, + "latitude": 40, + "longitude": 170.3, + } + + + The newly posted asset is returned in the response. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 201: CREATED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + print(sensor_data) + sensor = Sensor(**sensor_data) + db.session.add(sensor) + db.session.commit() + return sensor_schema.dump(sensor), 201 + + +""" +- fetch one +- patch name, attributes +- post one +- html left collapsing panel with these options. +asset_id = post_asset(args) +sensor_id = post_sensor(asset_id) +post_asset() +get_asset() +post_sensor() +get_sensor() +""" diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 68c836542..d256fd484 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -6,6 +6,7 @@ from flexmeasures import Sensor from flexmeasures.api.tests.utils import get_auth_token +from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data def test_fetch_one_sensor( @@ -37,3 +38,49 @@ def make_headers_for(user_email: str | None, client) -> dict: if user_email: headers["Authorization"] = get_auth_token(client, user_email, "testtest") return headers + + +def test_post_a_sensor(client, setup_api_test_data): + """ + Post one extra asset, as an admin user. + TODO: Soon we'll allow creating assets on an account-basis, i.e. for users + who have the user role "account-admin" or something similar. Then we'll + test that here. + """ + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + post_data = get_sensor_post_data() + print(post_data) + post_sensor_response = client.post( + url_for("SensorAPI:post"), + json=post_data, + headers={"content-type": "application/json", "Authorization": auth_token}, + ) + print("Server responded with:\n%s" % post_sensor_response.json) + assert post_sensor_response.status_code == 201 + assert post_sensor_response.json["name"] == "power" + + sensor: Sensor = Sensor.query.filter_by(name="power").one_or_none() + assert sensor is not None + assert sensor.unit == "kWh" + + +# db.session.query(GenericAsset) +# GenericAsset.query +# .filter(GenericAsset.name == "hoi") +# .filter_by(name="hoi") + + +# Sensor.query.filter(GenericAsset.name == "hoi") + +# .filter(Sensor.generic_asset_id == GenericAsset.id).join(GenericAsset) + +# .all() +# .one_or_none() +# .first() +# .count() + +# Sensor.query.join(GenericAsset).filter(GenericAsset.id==4).all() + +# Sensor.query.join(GenericAsset).filter(Sensor.generic_asset_id == GenericAsset.id, GenericAsset.account_id==2).all() + +# class GenericAsset(db.model) diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py index 41d4e8e5a..6b80b5312 100644 --- a/flexmeasures/api/v3_0/tests/utils.py +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -40,6 +40,16 @@ def get_asset_post_data(account_id: int = 1, asset_type_id: int = 1) -> dict: return post_data +def get_sensor_post_data(generic_asset_id: int = 1) -> dict: + post_data = { + "name": "power", + "event_resolution": 10, + "unit": "kWh", + "generic_asset_id": generic_asset_id, + } + return post_data + + def message_for_trigger_schedule( unknown_prices: bool = False, with_targets: bool = False, From 4e3f6c1f9bb2f5be05defcfd86c0861d6069243d Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 18 Jul 2023 13:49:44 +0200 Subject: [PATCH 03/24] post sensor still needs work Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 2 ++ .../api/v3_0/tests/test_sensors_api.py | 3 +- flexmeasures/api/v3_0/tests/utils.py | 2 +- flexmeasures/data/schemas/sensors.py | 6 +++- flexmeasures/data/schemas/times.py | 36 +++++++++++++++++++ 5 files changed, 46 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 6de2b480c..0078d959f 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -530,6 +530,7 @@ def fetch_one(self, id, sensor): :status 422: UNPROCESSABLE_ENTITY """ sensor = Sensor.query.filter(Sensor.id == 1).one_or_none() + sensor.resolution = sensor.event_resolution return sensor_schema.dump(sensor), 200 @route("", methods=["POST"]) @@ -569,6 +570,7 @@ def post(self, sensor_data: dict): :status 422: UNPROCESSABLE_ENTITY """ print(sensor_data) + sensor_data["event_resolution"] = sensor_data.pop("resolution") sensor = Sensor(**sensor_data) db.session.add(sensor) db.session.commit() diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index d256fd484..7d830d495 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -18,7 +18,8 @@ def test_fetch_one_sensor( "name": "some gas sensor", "unit": "m³/h", "entity_address": "ea1.2023-08.localhost:fm1.1", - "event_resolution": 10, + # "event_resolution": 10, #remove + "resolution": "PT10M", "generic_asset_id": 4, "timezone": "UTC", "status": 200, diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py index 6b80b5312..2240f07ec 100644 --- a/flexmeasures/api/v3_0/tests/utils.py +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -43,7 +43,7 @@ def get_asset_post_data(account_id: int = 1, asset_type_id: int = 1) -> dict: def get_sensor_post_data(generic_asset_id: int = 1) -> dict: post_data = { "name": "power", - "event_resolution": 10, + "resolution": "PT1H", "unit": "kWh", "generic_asset_id": generic_asset_id, } diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 30db92345..285bc9d2a 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -9,6 +9,7 @@ with_appcontext_if_needed, ) from flexmeasures.utils.unit_utils import is_valid_unit +from flexmeasures.data.schemas.times import NewDurationField class SensorSchemaMixin(Schema): @@ -28,7 +29,10 @@ class Meta: name = ma.auto_field(required=True) unit = ma.auto_field(required=True) timezone = ma.auto_field() - event_resolution = fields.TimeDelta(required=True, precision="minutes") + event_resolution = fields.TimeDelta(precision="minutes") + resolution = NewDurationField( + required=True + ) # fields.TimeDelta(required=True, precision="minutes") entity_address = fields.String(dump_only=True) @validates("unit") diff --git a/flexmeasures/data/schemas/times.py b/flexmeasures/data/schemas/times.py index f2f931356..aab9132e8 100644 --- a/flexmeasures/data/schemas/times.py +++ b/flexmeasures/data/schemas/times.py @@ -15,6 +15,42 @@ class DurationValidationError(FMValidationError): status = "INVALID_PERIOD" # USEF error status +class NewDurationField(MarshmallowClickMixin, fields.Str): + """Field that deserializes to a ISO8601 Duration + and serializes back to a string.""" + + def _deserialize(self, value, attr, obj, **kwargs) -> str: + """ + Use the isodate library to turn an ISO8601 string into a timedelta. + For some non-obvious cases, it will become an isodate.Duration, see + ground_from for more. + This method throws a ValidationError if the string is not ISO norm. + """ + try: + value_isodate = isodate.parse_duration(value) + except ISO8601Error as iso_err: + raise DurationValidationError( + f"Cannot parse {value} as ISO8601 duration: {iso_err}" + ) + + if value_isodate.seconds % 60 != 0 or value_isodate.microseconds != 0: + print(value_isodate.seconds) + print(value_isodate.microseconds) + raise DurationValidationError( + "FlexMeasures only support multiples of 1 minute." + ) + + return value + + def _serialize(self, value, attr, data, **kwargs): + """ + An implementation of _serialize. + It is not guaranteed to return the same string as was input, + if ground_from has been used! + """ + return isodate.strftime(value, "P%P") + + class DurationField(MarshmallowClickMixin, fields.Str): """Field that deserializes to a ISO8601 Duration and serializes back to a string.""" From 714b1a031c339cdffefd2f8ba72acbf0e00ddd32 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Thu, 20 Jul 2023 10:30:17 +0200 Subject: [PATCH 04/24] feat(sensor): adds post sensor Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 12 +++-- .../api/v3_0/tests/test_sensors_api.py | 50 ++++--------------- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 0078d959f..48a1e1f0b 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -529,9 +529,12 @@ def fetch_one(self, id, sensor): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - sensor = Sensor.query.filter(Sensor.id == 1).one_or_none() + sensor.resolution = sensor.event_resolution - return sensor_schema.dump(sensor), 200 + sensor_dict = sensor_schema.dump(sensor) + del sensor_dict["event_resolution"] + + return sensor_dict, 200 @route("", methods=["POST"]) @permission_required_for_context( @@ -574,7 +577,10 @@ def post(self, sensor_data: dict): sensor = Sensor(**sensor_data) db.session.add(sensor) db.session.commit() - return sensor_schema.dump(sensor), 201 + sensor.resolution = sensor.event_resolution + sensor_json = sensor_schema.dump(sensor) + del sensor_json["event_resolution"] + return sensor_json, 201 """ diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 7d830d495..bb7acc345 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -7,6 +7,9 @@ from flexmeasures import Sensor from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data +from flexmeasures.data.schemas.sensors import SensorSchema + +sensor_schema = SensorSchema() def test_fetch_one_sensor( @@ -14,24 +17,17 @@ def test_fetch_one_sensor( setup_api_test_data: dict[str, Sensor], ): sensor_id = 1 - assert_response = { - "name": "some gas sensor", - "unit": "m³/h", - "entity_address": "ea1.2023-08.localhost:fm1.1", - # "event_resolution": 10, #remove - "resolution": "PT10M", - "generic_asset_id": 4, - "timezone": "UTC", - "status": 200, - } headers = make_headers_for("test_supplier_user_4@seita.nl", client) response = client.get( url_for("SensorAPI:fetch_one", id=sensor_id), headers=headers, ) - print("Server responded with:\n%s" % response.json) assert response.status_code == 200 - assert response.json == assert_response + assert response.json["name"] == "some gas sensor" + assert response.json["unit"] == "m³/h" + assert response.json["generic_asset_id"] == 4 + assert response.json["timezone"] == "UTC" + assert response.json["resolution"] == "PT10M" def make_headers_for(user_email: str | None, client) -> dict: @@ -42,15 +38,8 @@ def make_headers_for(user_email: str | None, client) -> dict: def test_post_a_sensor(client, setup_api_test_data): - """ - Post one extra asset, as an admin user. - TODO: Soon we'll allow creating assets on an account-basis, i.e. for users - who have the user role "account-admin" or something similar. Then we'll - test that here. - """ auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") post_data = get_sensor_post_data() - print(post_data) post_sensor_response = client.post( url_for("SensorAPI:post"), json=post_data, @@ -59,29 +48,8 @@ def test_post_a_sensor(client, setup_api_test_data): print("Server responded with:\n%s" % post_sensor_response.json) assert post_sensor_response.status_code == 201 assert post_sensor_response.json["name"] == "power" + assert post_sensor_response.json["resolution"] == "PT1H" sensor: Sensor = Sensor.query.filter_by(name="power").one_or_none() assert sensor is not None assert sensor.unit == "kWh" - - -# db.session.query(GenericAsset) -# GenericAsset.query -# .filter(GenericAsset.name == "hoi") -# .filter_by(name="hoi") - - -# Sensor.query.filter(GenericAsset.name == "hoi") - -# .filter(Sensor.generic_asset_id == GenericAsset.id).join(GenericAsset) - -# .all() -# .one_or_none() -# .first() -# .count() - -# Sensor.query.join(GenericAsset).filter(GenericAsset.id==4).all() - -# Sensor.query.join(GenericAsset).filter(Sensor.generic_asset_id == GenericAsset.id, GenericAsset.account_id==2).all() - -# class GenericAsset(db.model) From 058cb41f280e58ad53107e045db83eac14ebc46d Mon Sep 17 00:00:00 2001 From: GustaafL Date: Thu, 20 Jul 2023 10:37:27 +0200 Subject: [PATCH 05/24] docs(sensor): changes the docstring of the post function Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 48a1e1f0b..5ae5a1e94 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -544,24 +544,23 @@ def fetch_one(self, id, sensor): def post(self, sensor_data: dict): """Create new asset. - .. :quickref: Asset; Create a new asset + .. :quickref: Sensor; Create a new Sensor - This endpoint creates a new asset. + This endpoint creates a new Sensor. **Example request** .. sourcecode:: json { - "name": "Test battery", - "generic_asset_type_id": 2, - "account_id": 2, - "latitude": 40, - "longitude": 170.3, + "name": "power", + "resolution": "PT1H", + "unit": "kWh", + "generic_asset_id": 1, } - The newly posted asset is returned in the response. + The newly posted sensor is returned in the response. :reqheader Authorization: The authentication token :reqheader Content-Type: application/json @@ -581,17 +580,3 @@ def post(self, sensor_data: dict): sensor_json = sensor_schema.dump(sensor) del sensor_json["event_resolution"] return sensor_json, 201 - - -""" -- fetch one -- patch name, attributes -- post one -- html left collapsing panel with these options. -asset_id = post_asset(args) -sensor_id = post_sensor(asset_id) -post_asset() -get_asset() -post_sensor() -get_sensor() -""" From 569140595637ffe268c33a3aebe52ce287ad133a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 20 Jul 2023 13:04:24 +0200 Subject: [PATCH 06/24] clearer names for the arguments to permission_required_for_context decorator, especially arg_loader was misleading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- .vscode/spellright.dict | 4 ++++ documentation/dev/auth.rst | 2 +- flexmeasures/api/dev/sensors.py | 10 ++++----- flexmeasures/api/v3_0/accounts.py | 2 +- flexmeasures/api/v3_0/assets.py | 12 +++++------ flexmeasures/api/v3_0/sensors.py | 8 ++++---- flexmeasures/api/v3_0/users.py | 8 ++++---- flexmeasures/auth/decorators.py | 34 +++++++++++++++---------------- 8 files changed, 42 insertions(+), 38 deletions(-) diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict index 0d4c10887..ab9117cd3 100644 --- a/.vscode/spellright.dict +++ b/.vscode/spellright.dict @@ -257,3 +257,7 @@ dataframe dataframes args docstrings +Auth +ctx_loader +ctx_arg_name +ctx_arg_pos diff --git a/documentation/dev/auth.rst b/documentation/dev/auth.rst index aeb4415da..4f8bb86d4 100644 --- a/documentation/dev/auth.rst +++ b/documentation/dev/auth.rst @@ -30,7 +30,7 @@ You, as the endpoint author, need to make sure this is checked. Here is an examp {"the_resource": ResourceIdField(data_key="resource_id")}, location="path", ) - @permission_required_for_context("read", arg_name="the_resource") + @permission_required_for_context("read", ctx_arg_name="the_resource") @as_json def view(resource_id: int, resource: Resource): return dict(name=resource.name) diff --git a/flexmeasures/api/dev/sensors.py b/flexmeasures/api/dev/sensors.py index 346ebfca9..a31465f46 100644 --- a/flexmeasures/api/dev/sensors.py +++ b/flexmeasures/api/dev/sensors.py @@ -50,7 +50,7 @@ class SensorAPI(FlaskView): }, location="query", ) - @permission_required_for_context("read", arg_name="sensor") + @permission_required_for_context("read", ctx_arg_name="sensor") def get_chart(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart @@ -85,7 +85,7 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs): }, location="query", ) - @permission_required_for_context("read", arg_name="sensor") + @permission_required_for_context("read", ctx_arg_name="sensor") def get_chart_data(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_data @@ -118,7 +118,7 @@ def get_chart_data(self, id: int, sensor: Sensor, **kwargs): }, location="query", ) - @permission_required_for_context("read", arg_name="sensor") + @permission_required_for_context("read", ctx_arg_name="sensor") def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): """GET from /sensor//chart_annotations @@ -147,7 +147,7 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs): {"sensor": SensorIdField(data_key="id")}, location="path", ) - @permission_required_for_context("read", arg_name="sensor") + @permission_required_for_context("read", ctx_arg_name="sensor") def get(self, id: int, sensor: Sensor): """GET from /sensor/ @@ -170,7 +170,7 @@ class AssetAPI(FlaskView): {"asset": AssetIdField(data_key="id")}, location="path", ) - @permission_required_for_context("read", arg_name="asset") + @permission_required_for_context("read", ctx_arg_name="asset") def get(self, id: int, asset: GenericAsset): """GET from /asset/ diff --git a/flexmeasures/api/v3_0/accounts.py b/flexmeasures/api/v3_0/accounts.py index 724332d39..21da1ff4b 100644 --- a/flexmeasures/api/v3_0/accounts.py +++ b/flexmeasures/api/v3_0/accounts.py @@ -70,7 +70,7 @@ def index(self): @route("/", methods=["GET"]) @use_kwargs({"account": AccountIdField(data_key="id")}, location="path") - @permission_required_for_context("read", arg_name="account") + @permission_required_for_context("read", ctx_arg_name="account") @as_json def get(self, id: int, account: Account): """API endpoint to get an account. diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index db505e34a..046e78f27 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -42,7 +42,7 @@ class AssetAPI(FlaskView): }, location="query", ) - @permission_required_for_context("read", arg_name="account") + @permission_required_for_context("read", ctx_arg_name="account") @as_json def index(self, account: Account): """List all assets owned by a certain account. @@ -144,7 +144,7 @@ def post(self, asset_data: dict): @route("/", methods=["GET"]) @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") - @permission_required_for_context("read", arg_name="asset") + @permission_required_for_context("read", ctx_arg_name="asset") @as_json def fetch_one(self, id, asset): """Fetch a given asset. @@ -180,7 +180,7 @@ def fetch_one(self, id, asset): @route("/", methods=["PATCH"]) @use_args(partial_asset_schema) @use_kwargs({"db_asset": AssetIdField(data_key="id")}, location="path") - @permission_required_for_context("update", arg_name="db_asset") + @permission_required_for_context("update", ctx_arg_name="db_asset") @as_json def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): """Update an asset given its identifier. @@ -236,7 +236,7 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): @route("/", methods=["DELETE"]) @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") - @permission_required_for_context("delete", arg_name="asset") + @permission_required_for_context("delete", ctx_arg_name="asset") @as_json def delete(self, id: int, asset: GenericAsset): """Delete an asset given its identifier. @@ -278,7 +278,7 @@ def delete(self, id: int, asset: GenericAsset): }, location="query", ) - @permission_required_for_context("read", arg_name="asset") + @permission_required_for_context("read", ctx_arg_name="asset") def get_chart(self, id: int, asset: GenericAsset, **kwargs): """GET from /assets//chart @@ -302,7 +302,7 @@ def get_chart(self, id: int, asset: GenericAsset, **kwargs): }, location="query", ) - @permission_required_for_context("read", arg_name="asset") + @permission_required_for_context("read", ctx_arg_name="asset") def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): """GET from /assets//chart_data diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 5ae5a1e94..d02c9c4ae 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -64,7 +64,7 @@ class SensorAPI(FlaskView): }, location="query", ) - @permission_required_for_context("read", arg_name="account") + @permission_required_for_context("read", ctx_arg_name="account") @as_json def index(self, account: Account): """API endpoint to list all sensors of an account. @@ -498,7 +498,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg @route("/", methods=["GET"]) @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path") - @permission_required_for_context("read", arg_name="sensor") + @permission_required_for_context("read", ctx_arg_name="sensor") @as_json def fetch_one(self, id, sensor): """Fetch a given sensor. @@ -537,10 +537,10 @@ def fetch_one(self, id, sensor): return sensor_dict, 200 @route("", methods=["POST"]) + @use_args(sensor_schema) @permission_required_for_context( - "create-children", arg_loader=AccountIdField.load_current + "create-children", ctx_arg_pos=0, ctx_arg_name="generic_asset_id" ) - @use_args(sensor_schema) def post(self, sensor_data: dict): """Create new asset. diff --git a/flexmeasures/api/v3_0/users.py b/flexmeasures/api/v3_0/users.py index d3e165fcf..459439954 100644 --- a/flexmeasures/api/v3_0/users.py +++ b/flexmeasures/api/v3_0/users.py @@ -44,7 +44,7 @@ class UserAPI(FlaskView): }, location="query", ) - @permission_required_for_context("read", arg_name="account") + @permission_required_for_context("read", ctx_arg_name="account") @as_json def index(self, account: Account, include_inactive: bool = False): """API endpoint to list all users of an account. @@ -90,7 +90,7 @@ def index(self, account: Account, include_inactive: bool = False): @route("/") @use_kwargs({"user": UserIdField(data_key="id")}, location="path") - @permission_required_for_context("read", arg_name="user") + @permission_required_for_context("read", ctx_arg_name="user") @as_json def get(self, id: int, user: UserModel): """API endpoint to get a user. @@ -128,7 +128,7 @@ def get(self, id: int, user: UserModel): @route("/", methods=["PATCH"]) @use_kwargs(partial_user_schema) @use_kwargs({"user": UserIdField(data_key="id")}, location="path") - @permission_required_for_context("update", arg_name="user") + @permission_required_for_context("update", ctx_arg_name="user") @as_json def patch(self, id: int, user: UserModel, **user_data): """API endpoint to patch user data. @@ -204,7 +204,7 @@ def patch(self, id: int, user: UserModel, **user_data): @route("//password-reset", methods=["PATCH"]) @use_kwargs({"user": UserIdField(data_key="id")}, location="path") - @permission_required_for_context("update", arg_name="user") + @permission_required_for_context("update", ctx_arg_name="user") @as_json def reset_user_password(self, id: int, user: UserModel): """API endpoint to reset the user's current password, cookies and auth tokens, and to email a password reset link to the user. diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index d2afbdd08..5fb9bc712 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -106,20 +106,20 @@ def decorated_view(*args, **kwargs): def permission_required_for_context( permission: str, - arg_pos: int | None = None, - arg_name: str | None = None, - arg_loader: Callable | None = None, + ctx_arg_pos: int | None = None, + ctx_arg_name: str | None = None, + ctx_loader: Callable | None = None, ): """ This decorator can be used to make sure that the current user has the necessary permission to access the context. The context needs to be an AuthModelMixin and is found ... - - by loading it via the arg_loader callable; + - by loading it via the ctx_loader callable; - otherwise: - * by the keyword argument arg_name; - * and/or by a position in the non-keyword arguments (arg_pos). - If nothing is passed, the context lookup defaults to arg_pos=0. + * by the keyword argument ctx_arg_name; + * and/or by a position in the non-keyword arguments (ctx_arg_pos). + If nothing is passed, the context lookup defaults to ctx_arg_pos=0. - Using both arg_name and arg_pos arguments is useful when Marshmallow de-serializes to a dict and you are using use_args. In this case, the context lookup applies first arg_pos, then arg_name. + Using both ctx_arg_name and ctx_arg_pos arguments is useful when Marshmallow de-serializes to a dict and you are using use_args. In this case, the context lookup applies first ctx_arg_pos, then ctx_arg_name. The permission needs to be a known permission and is checked with principal descriptions from the context's access control list (see AuthModelMixin.__acl__). @@ -130,7 +130,7 @@ def permission_required_for_context( {"the_resource": ResourceIdField(data_key="resource_id")}, location="path", ) - @permission_required_for_context("read", arg_name="the_resource") + @permission_required_for_context("read", ctx_arg_name="the_resource") @as_json def view(resource_id: int, the_resource: Resource): return dict(name=the_resource.name) @@ -145,14 +145,14 @@ def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): # load & check context - if arg_loader is not None: - context: AuthModelMixin = arg_loader() - elif arg_pos is not None and arg_name is not None: - context = args[arg_pos][arg_name] - elif arg_pos is not None: - context = args[arg_pos] - elif arg_name is not None: - context = kwargs[arg_name] + if ctx_loader is not None: + context: AuthModelMixin = ctx_loader() + elif ctx_arg_pos is not None and ctx_arg_name is not None: + context = args[ctx_arg_pos][ctx_arg_name] + elif ctx_arg_pos is not None: + context = args[ctx_arg_pos] + elif ctx_arg_name is not None: + context = kwargs[ctx_arg_name] else: context = args[0] From ed535750c1fa9fec8fd801f25a4a718ab848f8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 20 Jul 2023 13:05:40 +0200 Subject: [PATCH 07/24] one more renaming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v3_0/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 046e78f27..1ae1df869 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -103,7 +103,7 @@ def public(self): @route("", methods=["POST"]) @permission_required_for_context( - "create-children", arg_loader=AccountIdField.load_current + "create-children", ctx_arg_loader=AccountIdField.load_current ) @use_args(asset_schema) def post(self, asset_data: dict): From 237614869eff95146e35ecb920ff90e93f7d7367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 20 Jul 2023 13:56:59 +0200 Subject: [PATCH 08/24] expanding possibilities in the require_permission_for_context decorator, for when we only have an AuthModelMixin ID. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- .vscode/spellright.dict | 1 + flexmeasures/api/v3_0/assets.py | 2 +- flexmeasures/api/v3_0/sensors.py | 7 +++- flexmeasures/auth/decorators.py | 67 +++++++++++++++++++++++++------- flexmeasures/auth/policy.py | 2 +- 5 files changed, 61 insertions(+), 18 deletions(-) diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict index ab9117cd3..5b95070a8 100644 --- a/.vscode/spellright.dict +++ b/.vscode/spellright.dict @@ -261,3 +261,4 @@ Auth ctx_loader ctx_arg_name ctx_arg_pos +dataset diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 1ae1df869..ec240f03a 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -103,7 +103,7 @@ def public(self): @route("", methods=["POST"]) @permission_required_for_context( - "create-children", ctx_arg_loader=AccountIdField.load_current + "create-children", ctx_loader=AccountIdField.load_current ) @use_args(asset_schema) def post(self, asset_data: dict): diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index d02c9c4ae..ef6a8fb1b 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -30,6 +30,7 @@ from flexmeasures.auth.decorators import permission_required_for_context from flexmeasures.data import db from flexmeasures.data.models.user import Account +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField @@ -539,7 +540,11 @@ def fetch_one(self, id, sensor): @route("", methods=["POST"]) @use_args(sensor_schema) @permission_required_for_context( - "create-children", ctx_arg_pos=0, ctx_arg_name="generic_asset_id" + "create-children", + ctx_arg_pos=1, + ctx_arg_name="generic_asset_id", + ctx_loader=GenericAsset, + pass_ctx_to_loader=True, ) def post(self, sensor_data: dict): """Create new asset. diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 5fb9bc712..0624a06f6 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -109,9 +109,18 @@ def permission_required_for_context( ctx_arg_pos: int | None = None, ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, + pass_ctx_to_loader: bool = False, ): """ This decorator can be used to make sure that the current user has the necessary permission to access the context. + The permission needs to be a known permission and is checked with principal descriptions from the context's access control list (see AuthModelMixin.__acl__). + This decorator will first load the context (see below for details) and then call check_access to make sure the current user has the permission. + + A 403 response is raised if there is no principal for the required permission. + A 401 response is raised if the user is not authenticated at all. + + We will now explain how to load a context, and give an example: + The context needs to be an AuthModelMixin and is found ... - by loading it via the ctx_loader callable; - otherwise: @@ -119,11 +128,7 @@ def permission_required_for_context( * and/or by a position in the non-keyword arguments (ctx_arg_pos). If nothing is passed, the context lookup defaults to ctx_arg_pos=0. - Using both ctx_arg_name and ctx_arg_pos arguments is useful when Marshmallow de-serializes to a dict and you are using use_args. In this case, the context lookup applies first ctx_arg_pos, then ctx_arg_name. - - The permission needs to be a known permission and is checked with principal descriptions from the context's access control list (see AuthModelMixin.__acl__). - - Usually, you'd place a marshmallow field further up in the decorator chain, e.g.: + Let's look at an example. Usually, you'd place a marshmallow field further up in the decorator chain, e.g.: @app.route("/resource/", methods=["GET"]) @use_kwargs( @@ -135,26 +140,58 @@ def permission_required_for_context( def view(resource_id: int, the_resource: Resource): return dict(name=the_resource.name) - Where `ResourceIdField._deserialize()` turns the id parameter into a Resource context (if possible). + Note that in this example, `ResourceIdField._deserialize()` turns the id parameter into a Resource context (if possible). + + The ctx_loader: + + The ctx_loader can be a function without arguments or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). + A special case is useful when the arguments contain the context ID (not the instance). + Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. - This decorator raises a 403 response if there is no principal for the required permission. - It raises a 401 response if the user is not authenticated at all. + Using both arg name and position: + + Using both ctx_arg_name and ctx_arg_pos arguments is useful when Marshmallow de-serializes to a dict and you are using use_args. In this case, the context lookup applies first ctx_arg_pos, then ctx_arg_name. + + Let's look at a slightly more complex example where we combine both special cases from above. + We parse a dictionary from the input with a Marshmallow schema, in which a context ID can be found which we need to instantiate: + + @app.route("/resource", methods=["POST"]) + @use_args(resource_schema) + @permission_required_for_context( + "create-children", ctx_arg_pos=1, ctx_arg_name="resource_id", ctx_loader=Resource, pass_ctx_to_loader=True + ) + def post(self, resource_data: dict): + Note that in this example, resource_data is the input parsed by resource_schema, "resource_id" is one of the parameters in this schema, and Resource is a subclass of AuthModelMixin. """ def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): # load & check context - if ctx_loader is not None: - context: AuthModelMixin = ctx_loader() - elif ctx_arg_pos is not None and ctx_arg_name is not None: - context = args[ctx_arg_pos][ctx_arg_name] + context: AuthModelMixin = None + + # first set context_from_args, if possible + context_from_args: AuthModelMixin = None + if ctx_arg_pos is not None and ctx_arg_name is not None: + context_from_args = args[ctx_arg_pos][ctx_arg_name] elif ctx_arg_pos is not None: - context = args[ctx_arg_pos] + context_from_args = args[ctx_arg_pos] elif ctx_arg_name is not None: - context = kwargs[ctx_arg_name] + context_from_args = kwargs[ctx_arg_name] + elif len(args) > 0: + context_from_args = args[0] + + # if a loader is given, use that, otherwise fall back to context_from_args + if ctx_loader is not None: + if pass_ctx_to_loader: + if issubclass(ctx_loader, AuthModelMixin): + context = ctx_loader.query.get(context_from_args) + else: + context = ctx_loader(context_from_args) + else: + context = ctx_loader() else: - context = args[0] + context = context_from_args check_access(context, permission) diff --git a/flexmeasures/auth/policy.py b/flexmeasures/auth/policy.py index 6a2bbaa03..e85cf7f03 100644 --- a/flexmeasures/auth/policy.py +++ b/flexmeasures/auth/policy.py @@ -85,7 +85,7 @@ def check_access(context: AuthModelMixin, permission: str): Raises 401 or 403 otherwise. """ - # check current user + # check permission and current user before taking context into account if permission not in PERMISSIONS: raise Forbidden(f"Permission '{permission}' cannot be handled.") if current_user.is_anonymous: From 957c144a186292fe17865183c0498b0391a9e139 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Mon, 24 Jul 2023 09:05:18 +0200 Subject: [PATCH 09/24] feat(sensor): post sensor without schema changes Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 5 +---- flexmeasures/data/services/users.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index ef6a8fb1b..747767506 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -576,12 +576,9 @@ def post(self, sensor_data: dict): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ - print(sensor_data) sensor_data["event_resolution"] = sensor_data.pop("resolution") sensor = Sensor(**sensor_data) db.session.add(sensor) db.session.commit() sensor.resolution = sensor.event_resolution - sensor_json = sensor_schema.dump(sensor) - del sensor_json["event_resolution"] - return sensor_json, 201 + return sensor_schema.dump(sensor), 201 diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index 3a331a3d8..ed786d0e4 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -8,10 +8,10 @@ from flask_security.recoverable import update_password from email_validator import ( validate_email, - validate_email_deliverability, EmailNotValidError, EmailUndeliverableError, ) +from email_validator.deliverability import validate_email_deliverability from flask_security.utils import hash_password from werkzeug.exceptions import NotFound From 4d8739688a07cac5afa3fcee20224b0a00778c9c Mon Sep 17 00:00:00 2001 From: GustaafL Date: Mon, 24 Jul 2023 17:28:05 +0200 Subject: [PATCH 10/24] feat(sensor): adds patch sensor Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 64 +++++++++++++++++-- .../api/v3_0/tests/test_sensors_api.py | 13 +++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 747767506..8962fee8e 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -48,6 +48,7 @@ post_sensor_schema = PostSensorDataSchema() sensors_schema = SensorSchema(many=True) sensor_schema = SensorSchema() +partial_sensor_schema = SensorSchema(partial=True, exclude=["generic_asset_id"]) class SensorAPI(FlaskView): @@ -532,10 +533,8 @@ def fetch_one(self, id, sensor): """ sensor.resolution = sensor.event_resolution - sensor_dict = sensor_schema.dump(sensor) - del sensor_dict["event_resolution"] - - return sensor_dict, 200 + + return sensor_schema.dump(sensor), 200 @route("", methods=["POST"]) @use_args(sensor_schema) @@ -546,6 +545,7 @@ def fetch_one(self, id, sensor): ctx_loader=GenericAsset, pass_ctx_to_loader=True, ) + @as_json def post(self, sensor_data: dict): """Create new asset. @@ -582,3 +582,59 @@ def post(self, sensor_data: dict): db.session.commit() sensor.resolution = sensor.event_resolution return sensor_schema.dump(sensor), 201 + + + + @route("/", methods=["PATCH"]) + @use_args(partial_sensor_schema) + @use_kwargs({"db_sensor": SensorIdField(data_key="id")}, location="path") + @permission_required_for_context("update", ctx_arg_name="db_sensor") + @as_json + def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): + """Update an sensor given its identifier. + + .. :quickref: Sensor; Update a sensor + + This endpoint sets data for an existing sensor. + Any subset of sensor fields can be sent. + + The following fields are not allowed to be updated: + - id + - generic_asset_id + + **Example request** + + .. sourcecode:: json + + { + "name": "POWER", + } + + **Example response** + + The whole sensor is returned in the response: + + .. sourcecode:: json + + { + "name": "POWER", + "resolution": "PT1H", + "unit": "kWh", + "generic_asset_id": 1, + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: UPDATED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + for k, v in sensor_data.items(): + setattr(db_sensor, k, v) + db.session.add(db_sensor) + print(db_sensor) + db.session.commit() + return sensor_schema.dump(db_sensor), 200 diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index bb7acc345..acf1b2404 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -45,7 +45,7 @@ def test_post_a_sensor(client, setup_api_test_data): json=post_data, headers={"content-type": "application/json", "Authorization": auth_token}, ) - print("Server responded with:\n%s" % post_sensor_response.json) + assert post_sensor_response.status_code == 201 assert post_sensor_response.json["name"] == "power" assert post_sensor_response.json["resolution"] == "PT1H" @@ -53,3 +53,14 @@ def test_post_a_sensor(client, setup_api_test_data): sensor: Sensor = Sensor.query.filter_by(name="power").one_or_none() assert sensor is not None assert sensor.unit == "kWh" + + sensor_edit_response = client.patch( + url_for("SensorAPI:patch", id=sensor.id), + headers={"content-type": "application/json", "Authorization": auth_token}, + json={ + "name": "POWER", + }, + ) + + assert sensor_edit_response.json["name"] == "POWER" + From c0ba1108ee90085bc881e1478a48cd56cecd53d4 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 25 Jul 2023 11:59:00 +0200 Subject: [PATCH 11/24] feat(sensor): change importin users services Signed-off-by: GustaafL --- flexmeasures/data/services/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index ed786d0e4..3a331a3d8 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -8,10 +8,10 @@ from flask_security.recoverable import update_password from email_validator import ( validate_email, + validate_email_deliverability, EmailNotValidError, EmailUndeliverableError, ) -from email_validator.deliverability import validate_email_deliverability from flask_security.utils import hash_password from werkzeug.exceptions import NotFound From 5e6caa6861884632f12fb97333cc5e479d7a8ff6 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 25 Jul 2023 14:29:32 +0200 Subject: [PATCH 12/24] docs(sensor): update changelogs and fix import Signed-off-by: GustaafL --- documentation/api/change_log.rst | 5 +++++ documentation/changelog.rst | 1 + flexmeasures/data/services/users.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index b19fdf993..f529bba39 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,11 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-8 | 2023-07-25 +""""""""""""""""""" + +- Added REST endpoint for patching a sensor: `/sensor` (PATCH) + v3.0-10 | 2023-06-12 """""""""""""""""""" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 940ea1ccb..1d90ca182 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,6 +12,7 @@ New features * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 `_] * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 `_] * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 `_] +* Added API endpoint `/sensor/` for updating a sensor. [see `PR #773 `_] Bugfixes ----------- diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index 3a331a3d8..ed786d0e4 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -8,10 +8,10 @@ from flask_security.recoverable import update_password from email_validator import ( validate_email, - validate_email_deliverability, EmailNotValidError, EmailUndeliverableError, ) +from email_validator.deliverability import validate_email_deliverability from flask_security.utils import hash_password from werkzeug.exceptions import NotFound From 697270fde7e568a75e308f6a86ad2f3b1444261d Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 25 Jul 2023 15:01:48 +0200 Subject: [PATCH 13/24] feat(sensor): update failing test Signed-off-by: GustaafL --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index acf1b2404..aaeb2d8ac 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -54,6 +54,11 @@ def test_post_a_sensor(client, setup_api_test_data): assert sensor is not None assert sensor.unit == "kWh" + +def test_patch_sensor(client, setup_api_test_data): + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() + sensor_edit_response = client.patch( url_for("SensorAPI:patch", id=sensor.id), headers={"content-type": "application/json", "Authorization": auth_token}, @@ -63,4 +68,3 @@ def test_post_a_sensor(client, setup_api_test_data): ) assert sensor_edit_response.json["name"] == "POWER" - From 7c706e8ea48c7b1bb076d932a5594e6cb52b9b03 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 1 Aug 2023 17:29:40 +0200 Subject: [PATCH 14/24] feat(sensor): tests for patching excluded attributes and no auth Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 2 +- .../api/v3_0/tests/test_sensors_api.py | 43 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index a2609b8ef..e764276d8 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -611,7 +611,7 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): { "name": "POWER", - "resolution": "PT1H", + "event_resolution": "PT1H", "unit": "kWh", "generic_asset_id": 1, } diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 997b0d555..084405d7b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -112,11 +112,48 @@ def test_patch_sensor(client, setup_api_test_data): auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") sensor = Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() - sensor_edit_response = client.patch( + response = client.patch( url_for("SensorAPI:patch", id=sensor.id), headers={"content-type": "application/json", "Authorization": auth_token}, json={ - "name": "POWER", + "name": "Changed name", }, ) - assert sensor_edit_response.json["name"] == "POWER" + assert response.json["name"] == "Changed name" + + +def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data): + """Test to change the generic_asset_id that should not be allowed. + The generic_asset_id is excluded in the partial_sensor_schema""" + auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") + sensor = Sensor.query.filter(Sensor.name == "some temperature sensor").one_or_none() + + response = client.patch( + url_for("SensorAPI:patch", id=sensor.id), + headers={"content-type": "application/json", "Authorization": auth_token}, + json={ + "generic_asset_id": 8, + }, + ) + + assert response.status_code == 422 + assert response.json["status"] == "UNPROCESSABLE_ENTITY" + + +def test_patch_sensor_from_unrelated_account(client, setup_api_test_data): + """Try to change the name of a sensor that is in an account the user does not + have access to""" + headers = make_headers_for("test_prosumer_user_2@seita.nl", client) + + sensor = Sensor.query.filter(Sensor.name == "some temperature sensor").one_or_none() + + response = client.patch( + url_for("SensorAPI:patch", id=sensor.id), + headers=headers, + json={ + "name": "try to change the name", + }, + ) + + assert response.status_code == 403 + assert response.json["status"] == "INVALID_SENDER" From 5d425a3b0c0e64203a167fa4a3387b6e5ccfab31 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 1 Aug 2023 17:34:29 +0200 Subject: [PATCH 15/24] feat(sensor): remove NewDurationField since it's unused Signed-off-by: GustaafL --- flexmeasures/data/schemas/times.py | 36 ------------------------------ 1 file changed, 36 deletions(-) diff --git a/flexmeasures/data/schemas/times.py b/flexmeasures/data/schemas/times.py index 1acc4604c..8006eed26 100644 --- a/flexmeasures/data/schemas/times.py +++ b/flexmeasures/data/schemas/times.py @@ -17,42 +17,6 @@ class DurationValidationError(FMValidationError): status = "INVALID_PERIOD" # USEF error status -class NewDurationField(MarshmallowClickMixin, fields.Str): - """Field that deserializes to a ISO8601 Duration - and serializes back to a string.""" - - def _deserialize(self, value, attr, obj, **kwargs) -> str: - """ - Use the isodate library to turn an ISO8601 string into a timedelta. - For some non-obvious cases, it will become an isodate.Duration, see - ground_from for more. - This method throws a ValidationError if the string is not ISO norm. - """ - try: - value_isodate = isodate.parse_duration(value) - except ISO8601Error as iso_err: - raise DurationValidationError( - f"Cannot parse {value} as ISO8601 duration: {iso_err}" - ) - - if value_isodate.seconds % 60 != 0 or value_isodate.microseconds != 0: - print(value_isodate.seconds) - print(value_isodate.microseconds) - raise DurationValidationError( - "FlexMeasures only support multiples of 1 minute." - ) - - return value - - def _serialize(self, value, attr, data, **kwargs): - """ - An implementation of _serialize. - It is not guaranteed to return the same string as was input, - if ground_from has been used! - """ - return isodate.strftime(value, "P%P") - - class DurationField(MarshmallowClickMixin, fields.Str): """Field that deserializes to a ISO8601 Duration and serializes back to a string.""" From ddc9c78305f071ee794e67812851ea5103ce34cb Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 1 Aug 2023 17:35:47 +0200 Subject: [PATCH 16/24] feat(sensor): removes print statement Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index e764276d8..8f8b0cedc 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -628,6 +628,5 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): for k, v in sensor_data.items(): setattr(db_sensor, k, v) db.session.add(db_sensor) - print(db_sensor) db.session.commit() return sensor_schema.dump(db_sensor), 200 From 559593e62834a20f8e7cc78db8b3e74b515aec3d Mon Sep 17 00:00:00 2001 From: GustaafL Date: Tue, 1 Aug 2023 17:52:45 +0200 Subject: [PATCH 17/24] feat(sensor): test if database changed after patch Signed-off-by: GustaafL --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 084405d7b..592d18772 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -120,6 +120,9 @@ def test_patch_sensor(client, setup_api_test_data): }, ) assert response.json["name"] == "Changed name" + new_sensor = Sensor.query.filter(Sensor.name == "Changed name").one_or_none() + assert new_sensor.name == "Changed name" + assert Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() is None def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data): From 37d835915109354a7f592612d7502f9216e28f50 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 15:36:19 +0200 Subject: [PATCH 18/24] feat(sensor): tests for patch response json Signed-off-by: GustaafL --- documentation/api/change_log.rst | 14 +++----------- flexmeasures/api/v3_0/sensors.py | 2 +- flexmeasures/api/v3_0/tests/test_sensors_api.py | 7 +++++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 170a2a9f2..89bb18c2f 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,20 +5,12 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. -v3.0-8 | 2023-08-01 +v3.0-11 | 2023-08-02 """"""""""""""""""" -- Added REST endpoint for patching a sensor: `/sensor` (PATCH) - -v3.0-12 | 2023-07-31 -""""""""""""""""""" - -- Added REST endpoint for adding a sensor: `/sensors` (POST) - -v3.0-11 | 2023-07-20 -"""""""""""""""""""" - - Added REST endpoint for fetching one sensor: `/sensors/` (GET) +- Added REST endpoint for adding a sensor: `/sensors` (POST) +- Added REST endpoint for patching a sensor: `/sensor` (PATCH) v3.0-10 | 2023-06-12 """""""""""""""""""" diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 8f8b0cedc..6925c309f 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -584,7 +584,7 @@ def post(self, sensor_data: dict): @permission_required_for_context("update", ctx_arg_name="db_sensor") @as_json def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): - """Update an sensor given its identifier. + """Update a sensor given its identifier. .. :quickref: Sensor; Update a sensor diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 592d18772..7956f9879 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -125,7 +125,10 @@ def test_patch_sensor(client, setup_api_test_data): assert Sensor.query.filter(Sensor.name == "some gas sensor").one_or_none() is None -def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data): +@pytest.mark.parametrize( + "attribute", ["generic_asset_id", "timezone", "entity_address"] +) +def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data, attribute): """Test to change the generic_asset_id that should not be allowed. The generic_asset_id is excluded in the partial_sensor_schema""" auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") @@ -135,7 +138,7 @@ def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data): url_for("SensorAPI:patch", id=sensor.id), headers={"content-type": "application/json", "Authorization": auth_token}, json={ - "generic_asset_id": 8, + attribute: 8, }, ) From 1cdf47666132bf8f3958aa55ef465c75ea4f6093 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 15:49:00 +0200 Subject: [PATCH 19/24] docs(sensor): updates docstrings patch sensor Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 6925c309f..864b0fb33 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -594,6 +594,8 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): The following fields are not allowed to be updated: - id - generic_asset_id + - entity_address + - timezone **Example request** From 8c684ab86598160d68dda8e6651832d95344621d Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 16:15:11 +0200 Subject: [PATCH 20/24] tests(sensor): test for updating fields that are not allowed and improved docs Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 25 ++++++++++++++----- .../api/v3_0/tests/test_sensors_api.py | 11 +++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 864b0fb33..fbe565458 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -561,8 +561,20 @@ def post(self, sensor_data: dict): "generic_asset_id": 1, } + **Example response** + + The whole sensor is returned in the response: - The newly posted sensor is returned in the response. + .. sourcecode:: json + + { + "name": "power", + "unit": "kWh", + "entity_address": "ea1.2023-08.localhost:fm1.1", + "event_resolution": "PT1H", + "generic_asset_id": 1, + "timezone": "UTC", + } :reqheader Authorization: The authentication token :reqheader Content-Type: application/json @@ -595,7 +607,6 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): - id - generic_asset_id - entity_address - - timezone **Example request** @@ -612,10 +623,12 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): .. sourcecode:: json { - "name": "POWER", - "event_resolution": "PT1H", - "unit": "kWh", - "generic_asset_id": 1, + "name": "some gas sensor", + "unit": "m³/h", + "entity_address": "ea1.2023-08.localhost:fm1.1", + "event_resolution": "PT10M", + "generic_asset_id": 4, + "timezone": "UTC", } :reqheader Authorization: The authentication token diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 7956f9879..d67a3d219 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -126,9 +126,12 @@ def test_patch_sensor(client, setup_api_test_data): @pytest.mark.parametrize( - "attribute", ["generic_asset_id", "timezone", "entity_address"] + "attribute, value", + [("generic_asset_id", 8), ("entity_address", "ea1.2025-01.io.flexmeasures:fm1.1")], ) -def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data, attribute): +def test_patch_sensor_for_excluded_attribute( + client, setup_api_test_data, attribute, value +): """Test to change the generic_asset_id that should not be allowed. The generic_asset_id is excluded in the partial_sensor_schema""" auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest") @@ -138,12 +141,14 @@ def test_patch_sensor_for_excluded_attribute(client, setup_api_test_data, attrib url_for("SensorAPI:patch", id=sensor.id), headers={"content-type": "application/json", "Authorization": auth_token}, json={ - attribute: 8, + attribute: value, }, ) + print(response.json) assert response.status_code == 422 assert response.json["status"] == "UNPROCESSABLE_ENTITY" + assert response.json["message"]["json"][attribute] == ["Unknown field."] def test_patch_sensor_from_unrelated_account(client, setup_api_test_data): From c039fb998f35becd789a3dca3ac47091c95d3647 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 16:45:29 +0200 Subject: [PATCH 21/24] docs(sensor): updated api changelog typo Signed-off-by: GustaafL --- documentation/api/change_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 89bb18c2f..aac8e3694 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,7 +10,7 @@ v3.0-11 | 2023-08-02 - Added REST endpoint for fetching one sensor: `/sensors/` (GET) - Added REST endpoint for adding a sensor: `/sensors` (POST) -- Added REST endpoint for patching a sensor: `/sensor` (PATCH) +- Added REST endpoint for patching a sensor: `/sensors/` (PATCH) v3.0-10 | 2023-06-12 """""""""""""""""""" From c094558640ee045eb247e5e3f95dfa9f0e447555 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 20:28:58 +0200 Subject: [PATCH 22/24] feat(sensor): add id field(dump_only) to schema and response json Signed-off-by: GustaafL --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 6 +++++- flexmeasures/data/schemas/sensors.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index d67a3d219..7244be33e 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -127,7 +127,11 @@ def test_patch_sensor(client, setup_api_test_data): @pytest.mark.parametrize( "attribute, value", - [("generic_asset_id", 8), ("entity_address", "ea1.2025-01.io.flexmeasures:fm1.1")], + [ + ("generic_asset_id", 8), + ("entity_address", "ea1.2025-01.io.flexmeasures:fm1.1"), + ("id", 7), + ], ) def test_patch_sensor_for_excluded_attribute( client, setup_api_test_data, attribute, value diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 5bb7d1421..2826e6413 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -26,6 +26,7 @@ class Meta: model = Asset """ + id = ma.auto_field(dump_only=True) name = ma.auto_field(required=True) unit = ma.auto_field(required=True) timezone = ma.auto_field() From 3ca31cf3132c528203b83509aac9a12ab8277ee1 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Wed, 2 Aug 2023 21:14:26 +0200 Subject: [PATCH 23/24] docs(sensor): edit docstrings to include id in example response Signed-off-by: GustaafL --- flexmeasures/api/v3_0/sensors.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index fbe565458..be1d42fe1 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -86,11 +86,12 @@ def index(self, account: Account): [ { "entity_address": "ea1.2021-01.io.flexmeasures.company:fm1.42", - "event_resolution": 15, + "event_resolution": PT15M, "generic_asset_id": 1, "name": "Gas demand", "timezone": "Europe/Amsterdam", "unit": "m\u00b3/h" + "id": 2 } ] @@ -520,6 +521,7 @@ def fetch_one(self, id, sensor): "event_resolution": "PT10M", "generic_asset_id": 4, "timezone": "UTC", + "id": 2 } :reqheader Authorization: The authentication token @@ -574,6 +576,7 @@ def post(self, sensor_data: dict): "event_resolution": "PT1H", "generic_asset_id": 1, "timezone": "UTC", + "id": 2 } :reqheader Authorization: The authentication token @@ -629,6 +632,7 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): "event_resolution": "PT10M", "generic_asset_id": 4, "timezone": "UTC", + "id": 2 } :reqheader Authorization: The authentication token From ad1c345ecea292543ccc9607d6dabc4e9412c489 Mon Sep 17 00:00:00 2001 From: GustaafL <41048720+GustaafL@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:21:14 +0200 Subject: [PATCH 24/24] 433 delete sensor (#784) * feat(sensor): adds delete sensor endpoint Signed-off-by: GustaafL * docs(sensor): updated api changelog Signed-off-by: GustaafL * docs(sensor): updated api changelog whitespace Signed-off-by: GustaafL * docs(sensor): updated api changelog missing / Signed-off-by: GustaafL * docs(sensor): update changelog Signed-off-by: GustaafL * docs(sensor): update changelog typo Signed-off-by: GustaafL * docs(sensor): update changelog missing space Signed-off-by: GustaafL * tests(sensor): check for float instead of exact value Signed-off-by: GustaafL --------- Signed-off-by: GustaafL Signed-off-by: GustaafL <41048720+GustaafL@users.noreply.github.com> --- documentation/api/change_log.rst | 3 +- documentation/changelog.rst | 2 +- flexmeasures/api/v3_0/sensors.py | 33 ++++++++++++++++++- .../api/v3_0/tests/test_sensors_api.py | 27 ++++++++++++++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index aac8e3694..eedc66724 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,11 +6,12 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. v3.0-11 | 2023-08-02 -""""""""""""""""""" +"""""""""""""""""""" - Added REST endpoint for fetching one sensor: `/sensors/` (GET) - Added REST endpoint for adding a sensor: `/sensors` (POST) - Added REST endpoint for patching a sensor: `/sensors/` (PATCH) +- Added REST endpoint for deleting a sensor: `/sensors/` (DELETE) v3.0-10 | 2023-06-12 """""""""""""""""""" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2149619f9..63d58f95c 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,7 +18,7 @@ New features * Users on FlexMeasures servers in play mode (``FLEXMEASURES_MODE = "play"``) can use the ``sensors_to_show`` attribute to show any sensor on their asset pages, rather than only sensors registered to assets in their own account or to public assets [see `PR #740 `_] * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 `_] * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] -* Added API endpoints `/sensors/` for fetching a single sensor, `/sensors` (POST) for adding a sensor and `/sensor/` (PATCH) for updating a sensor. [see `PR #759 `_] and [see `PR #767 `_] and [see `PR #773 `_] +* Added API endpoints `/sensors/` for fetching a single sensor, `/sensors` (POST) for adding a sensor, `/sensors/` (PATCH) for updating a sensor and `/sensors/` (DELETE) for deleting a sensor. [see `PR #759 `_] and [see `PR #767 `_] and [see `PR #773 `_] and [see `PR #784 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] * Add `ProcessScheduler` class to optimize the starting time of processes one of the policies developed (INFLEXIBLE, SHIFTABLE and BREAKABLE), accessible via the CLI command `flexmeasures add schedule for-process` [see `PR #729 `_ and `PR #768 `_] diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index be1d42fe1..c9e3ecd62 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,5 +1,6 @@ from __future__ import annotations + from datetime import datetime, timedelta from flask import current_app @@ -31,7 +32,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField @@ -649,3 +650,33 @@ def patch(self, sensor_data: dict, id: int, db_sensor: Sensor): db.session.add(db_sensor) db.session.commit() return sensor_schema.dump(db_sensor), 200 + + @route("/", methods=["DELETE"]) + @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path") + @permission_required_for_context("delete", ctx_arg_name="sensor") + @as_json + def delete(self, id: int, sensor: Sensor): + """Delete a sensor given its identifier. + + .. :quickref: Sensor; Delete a sensor + + This endpoint deletes an existing sensor, as well as all measurements recorded for it. + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 204: DELETED + :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + + """Delete time series data.""" + TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).delete() + + sensor_name = sensor.name + db.session.delete(sensor) + db.session.commit() + current_app.logger.info("Deleted sensor '%s'." % sensor_name) + return {}, 204 diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index 7244be33e..03276b0ee 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -3,7 +3,7 @@ import pytest from flask import url_for - +from flexmeasures.data.models.time_series import TimedBelief from flexmeasures import Sensor from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data @@ -172,3 +172,28 @@ def test_patch_sensor_from_unrelated_account(client, setup_api_test_data): assert response.status_code == 403 assert response.json["status"] == "INVALID_SENDER" + + +def test_delete_a_sensor(client, setup_api_test_data): + + existing_sensor_id = setup_api_test_data["some temperature sensor"].id + headers = make_headers_for("test_admin_user@seita.nl", client) + sensor_data = TimedBelief.query.filter( + TimedBelief.sensor_id == existing_sensor_id + ).all() + sensor_count = len(Sensor.query.all()) + + assert isinstance(sensor_data[0].event_value, float) + + delete_sensor_response = client.delete( + url_for("SensorAPI:delete", id=existing_sensor_id), + headers=headers, + ) + assert delete_sensor_response.status_code == 204 + deleted_sensor = Sensor.query.filter_by(id=existing_sensor_id).one_or_none() + assert deleted_sensor is None + assert ( + TimedBelief.query.filter(TimedBelief.sensor_id == existing_sensor_id).all() + == [] + ) + assert len(Sensor.query.all()) == sensor_count - 1