From 844c0cac7126a3e1b522f51b8937275b08aeafc8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 12:41:20 +0200 Subject: [PATCH 01/13] Allow asset attributes in the UI (more specifically, the JSON attributes column on the generic_asset table) Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 16 ++++++++++++++++ flexmeasures/ui/crud/assets.py | 4 ++++ flexmeasures/ui/templates/crud/asset.html | 9 +++++++++ 3 files changed, 29 insertions(+) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 42d13d209..5035a921f 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -1,4 +1,5 @@ from typing import Optional +import json from marshmallow import validates, validates_schema, ValidationError, fields @@ -12,6 +13,20 @@ ) +class JSON(fields.Field): + def _deserialize(self, value, attr, data, **kwargs): + if value: + try: + return json.loads(value) + except ValueError: + return None + + return None + + def _serialize(self, value, attr, data, **kwargs): + return json.dumps(value) + + class GenericAssetSchema(ma.SQLAlchemySchema): """ GenericAsset schema, with validations. @@ -23,6 +38,7 @@ class GenericAssetSchema(ma.SQLAlchemySchema): latitude = ma.auto_field() longitude = ma.auto_field() generic_asset_type_id = fields.Integer(required=True) + attributes = JSON(required=False) class Meta: model = GenericAsset diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index a49339d97..19df9de21 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -1,5 +1,6 @@ from typing import Union, Optional, List, Tuple import copy +import json from flask import url_for, current_app from flask_classful import FlaskView @@ -47,6 +48,7 @@ class AssetForm(FlaskForm): places=4, render_kw={"placeholder": "--Click the map or enter a longitude--"}, ) + attributes = StringField("Attributes") def validate_on_submit(self): if ( @@ -125,7 +127,9 @@ def expunge_asset(): if asset_id: asset_data["id"] = asset_id if make_obj: + asset_data["attributes"] = json.loads(asset_data["attributes"]) asset = GenericAsset(**asset_data) # TODO: use schema? + asset_data["attributes"] = json.dumps(asset_data["attributes"]) asset.generic_asset_type = GenericAssetType.query.get( asset.generic_asset_type_id ) diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index a53655aea..5a6706300 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -61,6 +61,15 @@

Edit {{ asset.name }}

{% endfor %} +
+ {{ asset_form.attributes.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.attributes(class_="form-control") }} + {% for error in asset_form.errors.attributes %} + [{{error}}] + {% endfor %} +
+
{{ asset_form.latitude.label(class="col-sm-6 control-label") }}
From 1a0c1e085b596424ea3b58042050d15a4621993a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:13:52 +0200 Subject: [PATCH 02/13] Avoid redundant serialization step Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/assets.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index 19df9de21..a5194b063 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -127,9 +127,12 @@ def expunge_asset(): if asset_id: asset_data["id"] = asset_id if make_obj: - asset_data["attributes"] = json.loads(asset_data["attributes"]) - asset = GenericAsset(**asset_data) # TODO: use schema? - asset_data["attributes"] = json.dumps(asset_data["attributes"]) + asset = GenericAsset( + **{ + **asset_data, + **{"attributes": json.loads(asset_data["attributes"])}, + } + ) # TODO: use schema? asset.generic_asset_type = GenericAssetType.query.get( asset.generic_asset_type_id ) From fce9fefc88647b10b4b5161898d76b14887acbf8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:23:37 +0200 Subject: [PATCH 03/13] Move attributes field down Signed-off-by: F.N. Claessen --- flexmeasures/ui/templates/crud/asset.html | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index 5a6706300..6d4a342e6 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -61,15 +61,6 @@

Edit {{ asset.name }}

{% endfor %}
-
- {{ asset_form.attributes.label(class="col-sm-3 control-label") }} -
- {{ asset_form.attributes(class_="form-control") }} - {% for error in asset_form.errors.attributes %} - [{{error}}] - {% endfor %} -
-
{{ asset_form.latitude.label(class="col-sm-6 control-label") }}
@@ -95,7 +86,6 @@

Edit {{ asset.name }}

value="{{ asset.generic_asset_type.name }}" disabled>
-
@@ -103,6 +93,15 @@

Edit {{ asset.name }}

disabled>
+
+ {{ asset_form.attributes.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.attributes(class_="form-control") }} + {% for error in asset_form.errors.attributes %} + [{{error}}] + {% endfor %} +
+
(Click map to edit latitude and longitude in form)
From bbed3b426a346b9ba06d7686a1a9a758b641fc77 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:25:31 +0200 Subject: [PATCH 04/13] Rename field label Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index a5194b063..f0d2098e9 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -48,7 +48,7 @@ class AssetForm(FlaskForm): places=4, render_kw={"placeholder": "--Click the map or enter a longitude--"}, ) - attributes = StringField("Attributes") + attributes = StringField("Other attributes") def validate_on_submit(self): if ( From d13a1732fd34b54f6c9fc78d6cc4b5fd1254a2dd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:32:35 +0200 Subject: [PATCH 05/13] changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index a1a095c08..e089692df 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -29,6 +29,7 @@ Infrastructure / Support * Allow access tokens to be passed as env vars as well [see `PR #443 `_] * Queue workers can get initialised without a custom name and name collisions are handled [see `PR #455 `_] * New API endpoint to get public assets [see `PR #461 `_] +* Allow editing an asset's JSON attributes through the UI [see `PR #474 `_] v0.10.1 | June XX, 2022 From 38a1c1830a6090423aee2971f19ffd5d5856dc6e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:37:03 +0200 Subject: [PATCH 06/13] Add field to new asset page Signed-off-by: F.N. Claessen --- flexmeasures/ui/templates/crud/asset_new.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flexmeasures/ui/templates/crud/asset_new.html b/flexmeasures/ui/templates/crud/asset_new.html index 90011a317..72898f53e 100644 --- a/flexmeasures/ui/templates/crud/asset_new.html +++ b/flexmeasures/ui/templates/crud/asset_new.html @@ -70,6 +70,15 @@

Creating a new asset

{% endfor %} +
+ {{ asset_form.attributes.label(class="col-sm-6 control-label") }} +
+ {{ asset_form.attributes(class_="form-control") }} + {% for error in asset_form.errors.attributes %} + [{{error}}] + {% endfor %} +
+
From 733097e89587140392b5ab6defdd6b1b591f8be9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 11 Aug 2022 14:39:36 +0200 Subject: [PATCH 07/13] Add empty default attributes Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index f0d2098e9..4098a5a68 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -130,7 +130,7 @@ def expunge_asset(): asset = GenericAsset( **{ **asset_data, - **{"attributes": json.loads(asset_data["attributes"])}, + **{"attributes": json.loads(asset_data.get("attributes", "{}"))}, } ) # TODO: use schema? asset.generic_asset_type = GenericAssetType.query.get( From f79b80fc61b6214cda4488daa8ac33e43a930e05 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Aug 2022 17:51:54 +0200 Subject: [PATCH 08/13] Clarify attributes label Signed-off-by: F.N. Claessen --- flexmeasures/ui/crud/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index 4098a5a68..114c55e73 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -48,7 +48,7 @@ class AssetForm(FlaskForm): places=4, render_kw={"placeholder": "--Click the map or enter a longitude--"}, ) - attributes = StringField("Other attributes") + attributes = StringField("Other attributes (JSON)") def validate_on_submit(self): if ( From 8a53b922675e2506e487da873533415cdeea4028 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Aug 2022 18:00:44 +0200 Subject: [PATCH 09/13] Raise ValidationError if not able to deserialize JSON string Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 5035a921f..45188ecdb 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -15,13 +15,12 @@ class JSON(fields.Field): def _deserialize(self, value, attr, data, **kwargs): - if value: - try: - return json.loads(value) - except ValueError: - return None - - return None + try: + return json.loads(value) + except ValueError: + raise ValidationError( + f"Not a valid JSON string.", + ) def _serialize(self, value, attr, data, **kwargs): return json.dumps(value) From 981e7bff9808ceebd71b5bf1930c65dc0e53cc98 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Aug 2022 21:13:14 +0200 Subject: [PATCH 10/13] Update documentation for asset page Signed-off-by: F.N. Claessen --- documentation/views/asset-data.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index a9cec351a..073e0786f 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -4,7 +4,10 @@ Assets & data ************** -The asset page allows to see data from the asset's sensors, and also to edit meta information, like its location: +The asset page allows to see data from the asset's sensors, and also to edit attributes of the asset, like its location. +Other attributes are stored as a JSON string, which can be edited here as well. +This is meant for meta information that may be used to customize views or functionality, e.g. by plugins. +This includes the possibility to specify which sensors the asset page should show. For instance, here we include a price sensor from a public asset, by setting ``{"sensor_to_show": [3, 2]}`` (sensor 3 on top, followed by sensor 2 below). .. image:: https://github.com/FlexMeasures/screenshots/raw/main/screenshot_asset.png @@ -14,6 +17,6 @@ The asset page allows to see data from the asset's sensors, and also to edit met | | -.. note:: It is possible to determine which sensors this page should show. For instance, here we include a price sensor from a public asset. - For this, set the `sensors_to_show` attribute with the CLI command ``flexmeasures edit attribute``. +.. note:: While it is possible to show an arbitrary number of sensors this way, we recommend showing only the most crucial ones for faster loading, less page scrolling, and generally, a quick grasp of what the asset is up to. +.. note:: Asset attributes can be edited through the CLI as well, with the CLI command ``flexmeasures edit attribute``. From 1aa09274dc1730224af59183bef9312cd4c52c2a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Aug 2022 21:14:29 +0200 Subject: [PATCH 11/13] flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 45188ecdb..3b7cc8ed6 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -18,9 +18,7 @@ def _deserialize(self, value, attr, data, **kwargs): try: return json.loads(value) except ValueError: - raise ValidationError( - f"Not a valid JSON string.", - ) + raise ValidationError("Not a valid JSON string.") def _serialize(self, value, attr, data, **kwargs): return json.dumps(value) From be8263ba942c3c06fca6e3b81b96f9f2bdcb67f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Aug 2022 21:29:46 +0200 Subject: [PATCH 12/13] Add tests Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_assets_api.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 56ee1da38..b8572eb46 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -1,3 +1,5 @@ +import json + from flask import url_for import pytest @@ -144,6 +146,50 @@ def test_alter_an_asset(client, setup_api_test_data, setup_accounts): assert asset_edit_response.status_code == 200 +@pytest.mark.parametrize( + "bad_json_str", + [ + None, + "{", + '{"hallo": world}', + ], +) +def test_alter_an_asset_with_bad_json_attributes( + client, setup_api_test_data, setup_accounts, bad_json_str +): + """Check whether updating an asset's attributes with a badly structured JSON fails.""" + with UserContext("test_prosumer_user@seita.nl") as prosumer1: + auth_token = prosumer1.get_auth_token() + with AccountContext("Test Prosumer Account") as prosumer: + prosumer_asset = prosumer.generic_assets[0] + asset_edit_response = client.patch( + url_for("AssetAPI:patch", id=prosumer_asset.id), + headers={"content-type": "application/json", "Authorization": auth_token}, + json={"attributes": bad_json_str}, + ) + print(f"Editing Response: {asset_edit_response.json}") + assert asset_edit_response.status_code == 422 + + +def test_alter_an_asset_with_json_attributes( + client, setup_api_test_data, setup_accounts +): + """Check whether updating an asset's attributes with a properly structured JSON succeeds.""" + with UserContext("test_prosumer_user@seita.nl") as prosumer1: + auth_token = prosumer1.get_auth_token() + with AccountContext("Test Prosumer Account") as prosumer: + prosumer_asset = prosumer.generic_assets[0] + asset_edit_response = client.patch( + url_for("AssetAPI:patch", id=prosumer_asset.id), + headers={"content-type": "application/json", "Authorization": auth_token}, + json={ + "attributes": json.dumps(prosumer_asset.attributes) + }, # we're not changing values to keep other tests clean here + ) + print(f"Editing Response: {asset_edit_response.json}") + assert asset_edit_response.status_code == 200 + + def test_post_an_asset_with_existing_name(client, setup_api_test_data): """Catch DB error (Unique key violated) correctly""" with UserContext("test_admin_user@seita.nl") as admin_user: From e30bd8e28138eff40f86ec00afdb6ffa08404ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 25 Aug 2022 14:31:24 +0200 Subject: [PATCH 13/13] improve error visualisation in asset form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/ui/crud/assets.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets.py index 114c55e73..14ea9dbcf 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets.py @@ -292,11 +292,14 @@ def post(self, id: str): f"Internal asset API call unsuccessful [{post_asset_response.status_code}]: {post_asset_response.text}" ) asset_form.process_api_validation_errors(post_asset_response.json()) - if ( - "message" in post_asset_response.json() - and "json" in post_asset_response.json()["message"] - ): - error_msg = str(post_asset_response.json()["message"]["json"]) + if "message" in post_asset_response.json(): + asset_form.process_api_validation_errors( + post_asset_response.json()["message"] + ) + if "json" in post_asset_response.json()["message"]: + error_msg = str( + post_asset_response.json()["message"]["json"] + ) if asset is None: msg = "Cannot create asset. " + error_msg return render_flexmeasures_template( @@ -346,7 +349,9 @@ def post(self, id: str): f"Internal asset API call unsuccessful [{patch_asset_response.status_code}]: {patch_asset_response.text}" ) msg = "Cannot edit asset." - asset_form.process_api_validation_errors(patch_asset_response.json()) + asset_form.process_api_validation_errors( + patch_asset_response.json().get("message") + ) asset = GenericAsset.query.get(id) latest_measurement_time_str, asset_plot_html = _get_latest_power_plot(asset)