From 6ca213462c25a445e78dc6ad3b99356c10de041d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 21 Feb 2021 21:12:00 +0100 Subject: [PATCH 01/12] first working version of a charts/power view; also move Duration definition to a full marshmallow field definition --- flexmeasures/api/common/schemas.py | 29 +++++ flexmeasures/api/common/utils/validators.py | 15 +-- flexmeasures/app.py | 12 +- flexmeasures/ui/__init__.py | 4 + flexmeasures/ui/views/__init__.py | 2 + flexmeasures/ui/views/analytics.py | 13 +- flexmeasures/ui/views/charts.py | 129 ++++++++++++++++++++ 7 files changed, 183 insertions(+), 21 deletions(-) create mode 100644 flexmeasures/api/common/schemas.py create mode 100644 flexmeasures/ui/views/charts.py diff --git a/flexmeasures/api/common/schemas.py b/flexmeasures/api/common/schemas.py new file mode 100644 index 000000000..5caa55665 --- /dev/null +++ b/flexmeasures/api/common/schemas.py @@ -0,0 +1,29 @@ +from typing import Union + +from datetime import timedelta + +from marshmallow import fields, ValidationError +import isodate +from isodate.isoerror import ISO8601Error + + +class PeriodField(fields.Str): + """Field that serializes to a Period and deserializes to a string.""" + + def _deserialize( + self, value, attr, obj, **kwargs + ) -> Union[timedelta, isodate.Duration]: + return isodate.parse_duration(value) + + def _vaidate(value): + """Validate a marshmallow ISO8601 duration field, + throw marshmallow validation error if it cannot be parsed.""" + try: + isodate.parse_duration(value) + except ISO8601Error as iso_err: + raise ValidationError( + f"Cannot parse {value} as ISO8601 duration: {iso_err}" + ) + + def _serialize(self, value, attr, data, **kwargs): + return isodate.strftime(value, "%P") diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index d1cd4da10..150a89e41 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -14,9 +14,9 @@ from flask_security import current_user import marshmallow -from webargs import fields from webargs.flaskparser import parser +from flexmeasures.api.common.schemas import PeriodField from flexmeasures.api.common.responses import ( # noqa: F401 required_info_missing, invalid_horizon, @@ -147,17 +147,6 @@ def parse_duration( return None -def validate_duration_field(duration_str): - """Validate a marshmallow ISO8601 duration field, - throw marshmallow validation error if it cannot be parsed.""" - try: - isodate.parse_duration(duration_str) - except ISO8601Error as iso_err: - raise marshmallow.ValidationError( - f"Cannot parse {duration_str} as ISO8601 duration: {iso_err}" - ) - - def parse_isodate_str(start: str) -> Union[datetime, None]: """ Validates whether the string 'start' is a valid ISO 8601 datetime. @@ -203,7 +192,7 @@ def wrapper(fn): @as_json def decorated_service(*args, **kwargs): duration_arg = parser.parse( - {"duration": fields.Str(validate=validate_duration_field)}, + {"duration": PeriodField()}, request, location="args_and_json", unknown=marshmallow.EXCLUDE, diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 8e785b789..989f1d2ba 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -80,18 +80,18 @@ def create(env=None) -> Flask: register_db_at(app) - # Register the UI - - from flexmeasures.ui import register_at as register_ui_at - - register_ui_at(app) - # Register the API from flexmeasures.api import register_at as register_api_at register_api_at(app) + # Register the UI + + from flexmeasures.ui import register_at as register_ui_at + + register_ui_at(app) + # Profile endpoints (if needed, e.g. during development) @app.before_request def before_request(): diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index e37c2a5c1..a7c4e6639 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -15,6 +15,7 @@ localized_datetime_str, naturalized_datetime_str, ) +from flexmeasures.api.v2_0 import flexmeasures_api as flexmeasures_api_v2_0 # The ui blueprint. It is registered with the Flask app (see app.py) flexmeasures_ui = Blueprint( @@ -57,6 +58,9 @@ def favicon(): add_jinja_filters(app) add_jinja_variables(app) + # re-register api blueprint so it'll register the chart views (see views.charts) + app.register_blueprint(flexmeasures_api_v2_0, first_registration=False) + def register_rq_dashboard(app): app.config.update( diff --git a/flexmeasures/ui/views/__init__.py b/flexmeasures/ui/views/__init__.py index c4441ae5b..44b39ca86 100644 --- a/flexmeasures/ui/views/__init__.py +++ b/flexmeasures/ui/views/__init__.py @@ -11,6 +11,8 @@ from flexmeasures.ui.views.account import account_view # noqa: F401 # noqa: F401 +from flexmeasures.ui.views.charts import get_power_chart # noqa: F401 + @flexmeasures_ui.route("/docs") def docs_view(): diff --git a/flexmeasures/ui/views/analytics.py b/flexmeasures/ui/views/analytics.py index 1b95949e3..501f97db4 100644 --- a/flexmeasures/ui/views/analytics.py +++ b/flexmeasures/ui/views/analytics.py @@ -535,6 +535,16 @@ def make_power_figure( else: title = "Electricity production from %s" % resource_display_name + resolution_str = "?" + if data.index.freq is not None: + resolution_str = time_utils.freq_label_to_human_readable_label( + data.index.freqstr + ) + elif "resolution" in session: + resolution_str = time_utils.freq_label_to_human_readable_label( + session["resolution"] + ) + return create_graph( data, unit="MW", @@ -546,8 +556,7 @@ def make_power_figure( schedules=schedule_data, title=title, x_range=shared_x_range, - x_label="Time (resolution of %s)" - % time_utils.freq_label_to_human_readable_label(session["resolution"]), + x_label="Time (resolution of %s)" % resolution_str, y_label="Power (in MW)", show_y_floats=True, tools=tools, diff --git a/flexmeasures/ui/views/charts.py b/flexmeasures/ui/views/charts.py new file mode 100644 index 000000000..78eba1d62 --- /dev/null +++ b/flexmeasures/ui/views/charts.py @@ -0,0 +1,129 @@ +from flask_security import roles_accepted +from bokeh.embed import json_item +from marshmallow import Schema, fields +from webargs.flaskparser import use_args +from flask_cors import cross_origin +from flask_json import as_json + +from flexmeasures.api.v2_0 import flexmeasures_api as flexmeasures_api_v2_0 +from flexmeasures.api.v2_0.routes import v2_0_service_listing +from flexmeasures.api.common.schemas import PeriodField +from flexmeasures.data.queries.analytics import get_power_data +from flexmeasures.ui.views.analytics import make_power_figure + + +""" +An endpoint to get a power chart. + +This will grow to become code for more charts eventually. +The plan is to separate charts specs from the actual data later, +and to switch to Altair. + +For now, we'll keep this endpoint here, with route and implementation in the same file. +When we move forward, we'll review the architecture. +""" + + +v2_0_service_listing["services"].append( + { + "name": "POST /charts/power", + "access": ["admin", "Prosumer"], + "description": "Get a Bokeh chart for power data to embed in web pages.", + }, +) + + +class ChartRequestSchema(Schema): + """ + This schema describes the request for a chart. + """ + + resource = fields.Str(required=True) + start_time = fields.DateTime(required=True) + end_time = fields.DateTime(required=True) + resolution = PeriodField(required=True) + show_consumption_as_positive = fields.Bool(missing=True) + forecast_horizon = PeriodField(missing="PT6H") + + +@flexmeasures_api_v2_0.route("/charts/power", methods=["POST"]) +@cross_origin() # origins=["localhost"]) +@roles_accepted("admin", "Prosumer") +@use_args(ChartRequestSchema()) +@as_json +def get_power_chart(chart_request): + """API endpoint to get a chart for power data which can be embedded in web pages. + + .. :quickref: Chart; Get a power chart + + This endpoint returns a Bokeh chart with power data which can be embedded in a website. + It includes forecasts and even schedules, if available. + + **Example request** + + An example of a chart request: + + .. sourcecode:: json + + { + "resource": ""my-battery, + "start_time": "2020-02-20:10:00:00UTC", + "end_time": "2020-02-20:11:00:00UTC", + "resolution": "PT15M", + "consumption_as_positive": true + "resolution": "PT6H", + } + + On your webpage, you need to include the Bokeh libraries, e.g.: + + + + (The version needs to match the version used by the FlexMeasures server, see requirements/app.txt) + + Then you can call this endpoint and include the result like this: + + .. sourcecode:: javascript + + + + where `data` contains the chart request parameters (see above). + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_REQUEST + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 422: UNPROCESSABLE_ENTITY + """ + data = get_power_data( + resource=chart_request["resource"], + show_consumption_as_positive=chart_request["show_consumption_as_positive"], + showing_individual_traces_for="none", # TODO: parameterise + metrics={}, # will be stored here, we don't need them for now + query_window=(chart_request["start_time"], chart_request["end_time"]), + resolution=chart_request["resolution"], + forecast_horizon=chart_request["forecast_horizon"], + ) + figure = make_power_figure( + resource_display_name=chart_request["resource"], + data=data[0], + forecast_data=data[1], + schedule_data=data[2], + show_consumption_as_positive=chart_request["show_consumption_as_positive"], + shared_x_range=None, + ) + return json_item(figure) From 4f4d0347cfac26ae21626d2b0ac5fb1504439800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 21 Feb 2021 21:59:02 +0100 Subject: [PATCH 02/12] add flask-cors as requirement so we can allow endpoints to be reached from other domains --- requirements/app.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/app.in b/requirements/app.in index 71d979b7c..a4caae833 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -43,7 +43,8 @@ Flask-Mail Flask-Security-Too Flask-Classful Flask-Marshmallow +Flask-Cors marshmallow-sqlalchemy>=0.23.1 webargs # flask should be after all the flask plugins, because setup might find they ARE flask -flask>=1.0 \ No newline at end of file +flask>=1.0 From e88824d43772066db5964cab3ca925d944d0e717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 23 Feb 2021 00:19:19 +0100 Subject: [PATCH 03/12] full implementattion of DurationField, including the logic to ground by a start time. Adding tests for DurationField, as well. --- flexmeasures/api/common/schemas.py | 45 +++++++++---- flexmeasures/api/common/tests/__init__.py | 0 flexmeasures/api/common/tests/test_schemas.py | 64 +++++++++++++++++++ flexmeasures/api/common/utils/validators.py | 24 +++---- flexmeasures/ui/views/charts.py | 6 +- 5 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 flexmeasures/api/common/tests/__init__.py create mode 100644 flexmeasures/api/common/tests/test_schemas.py diff --git a/flexmeasures/api/common/schemas.py b/flexmeasures/api/common/schemas.py index 5caa55665..1412a5790 100644 --- a/flexmeasures/api/common/schemas.py +++ b/flexmeasures/api/common/schemas.py @@ -1,29 +1,50 @@ -from typing import Union - -from datetime import timedelta +from typing import Union, Optional +from datetime import datetime, timedelta from marshmallow import fields, ValidationError import isodate from isodate.isoerror import ISO8601Error -class PeriodField(fields.Str): - """Field that serializes to a Period and deserializes to a string.""" +class DurationField(fields.Str): + """Field that deserializes to a ISO8601 Duration + and serializes back to a string.""" def _deserialize( self, value, attr, obj, **kwargs ) -> Union[timedelta, isodate.Duration]: - return isodate.parse_duration(value) - - def _vaidate(value): - """Validate a marshmallow ISO8601 duration field, - throw marshmallow validation error if it cannot be parsed.""" + """ + Use the isodate library to turn an ISO8601 string into a timedelta. + For some non-obvious cases, it will become an isodate.Duration, see + ground_from for more. + This method throws a ValidationError if the string is not ISO norm. + """ try: - isodate.parse_duration(value) + return isodate.parse_duration(value) except ISO8601Error as iso_err: raise ValidationError( f"Cannot parse {value} as ISO8601 duration: {iso_err}" ) def _serialize(self, value, attr, data, **kwargs): - return isodate.strftime(value, "%P") + """ + An implementation of _serialize. + It is not guaranteed to return the same string as was input, + if ground_from has been used! + """ + return isodate.strftime(value, "P%P") + + @staticmethod + def ground_from( + duration: Union[timedelta, isodate.Duration], start: Optional[datetime] + ) -> timedelta: + """ + For some valid duration strings (such as "P1M", a month), + converting to a datetime.timedelta is not possible (no obvious + number of days). In this case, `_deserialize` returned an + `isodate.Duration`. We can derive the timedelta by grounding to an + actual time span, for which we require a start datetime. + """ + if isinstance(duration, isodate.Duration) and start: + return (start + duration) - start + return duration diff --git a/flexmeasures/api/common/tests/__init__.py b/flexmeasures/api/common/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/api/common/tests/test_schemas.py b/flexmeasures/api/common/tests/test_schemas.py new file mode 100644 index 000000000..ed953682b --- /dev/null +++ b/flexmeasures/api/common/tests/test_schemas.py @@ -0,0 +1,64 @@ +from datetime import datetime, timedelta + +import pytest +import isodate +from marshmallow import ValidationError + +from flexmeasures.api.common.schemas import DurationField + + +@pytest.mark.parametrize( + "duration_input,exp_deserialization", + [ + ("PT1H", timedelta(hours=1)), + ("PT6M", timedelta(minutes=6)), + ("PT6H", timedelta(hours=6)), + ("P2DT1H", timedelta(hours=49)), + ], +) +def test_duration_field_straightforward(duration_input, exp_deserialization): + """Testing straightforward cases""" + df = DurationField() + deser = df.deserialize(duration_input, None, None) + assert deser == exp_deserialization + assert df.serialize("duration", {"duration": deser}) == duration_input + + +@pytest.mark.parametrize( + "duration_input,exp_deserialization,grounded_timedelta", + [ + ("P1M", isodate.Duration(months=1), timedelta(days=29)), + ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)), + ("P2D", isodate.Duration(hours=48), timedelta(hours=48)), + ], +) +def test_duration_field_nominal_grounded( + duration_input, exp_deserialization, grounded_timedelta +): + """Nominal durations are tricky: + https://en.wikipedia.org/wiki/Talk:ISO_8601/Archive_2#Definition_of_Duration_is_incorrect + We want to test if we can ground them as expected. + We use a particular datetime to ground, in a leap year february. + """ + df = DurationField() + deser = df.deserialize(duration_input, None, None) + assert deser == exp_deserialization + dummy_time = datetime(2020, 2, 22, 18, 7) + grounded = DurationField.ground_from(deser, dummy_time) + assert grounded == grounded_timedelta + + +@pytest.mark.parametrize( + "duration_input,error_msg", + [ + ("", "Unable to parse duration string"), + ("1H", "Unable to parse duration string"), + ("PP1M", "time designator 'T' missing"), + ("PT2D", "Unrecognised ISO 8601 date format"), + ], +) +def test_duration_field_invalid(duration_input, error_msg): + df = DurationField() + with pytest.raises(ValidationError) as ve: + df.deserialize(duration_input, None, None) + assert error_msg in str(ve) diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index 150a89e41..b973e4a20 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -16,7 +16,7 @@ from webargs.flaskparser import parser -from flexmeasures.api.common.schemas import PeriodField +from flexmeasures.api.common.schemas import DurationField from flexmeasures.api.common.responses import ( # noqa: F401 required_info_missing, invalid_horizon, @@ -134,15 +134,15 @@ def parse_duration( Parses the 'duration' string into a Duration object. If needed, try deriving the timedelta from the actual time span (e.g. in case duration is 1 year). If the string is not a valid ISO 8601 time interval, return None. + + TODO: Deprecate for DurationField. """ try: duration = isodate.parse_duration(duration_str) - if not isinstance(duration, timedelta): - if start: - return (start + duration) - start - return duration # valid duration, but not a timedelta (e.g. "P1Y" could be leap year) - else: - return isodate.parse_duration(duration_str) + if not isinstance(duration, timedelta) and start: + return (start + duration) - start + # if not a timedelta, then it's a valid duration (e.g. "P1Y" could be leap year) + return duration except (ISO8601Error, AttributeError): return None @@ -192,18 +192,18 @@ def wrapper(fn): @as_json def decorated_service(*args, **kwargs): duration_arg = parser.parse( - {"duration": PeriodField()}, + {"duration": DurationField()}, request, location="args_and_json", unknown=marshmallow.EXCLUDE, ) - if "duration" in duration_arg: - duration = parse_duration( - duration_arg["duration"], + duration = duration_arg["duration"] + duration = DurationField.ground_from( + duration, kwargs.get("start", kwargs.get("datetime", None)), ) - if not duration: + if not duration: # TODO: deprecate extra_info = "Cannot parse 'duration' value." current_app.logger.warning(extra_info) return invalid_period(extra_info) diff --git a/flexmeasures/ui/views/charts.py b/flexmeasures/ui/views/charts.py index 78eba1d62..f299cc47f 100644 --- a/flexmeasures/ui/views/charts.py +++ b/flexmeasures/ui/views/charts.py @@ -7,7 +7,7 @@ from flexmeasures.api.v2_0 import flexmeasures_api as flexmeasures_api_v2_0 from flexmeasures.api.v2_0.routes import v2_0_service_listing -from flexmeasures.api.common.schemas import PeriodField +from flexmeasures.api.common.schemas import DurationField from flexmeasures.data.queries.analytics import get_power_data from flexmeasures.ui.views.analytics import make_power_figure @@ -41,9 +41,9 @@ class ChartRequestSchema(Schema): resource = fields.Str(required=True) start_time = fields.DateTime(required=True) end_time = fields.DateTime(required=True) - resolution = PeriodField(required=True) + resolution = DurationField(required=True) show_consumption_as_positive = fields.Bool(missing=True) - forecast_horizon = PeriodField(missing="PT6H") + forecast_horizon = DurationField(missing="PT6H") # TODO: HorizonField @flexmeasures_api_v2_0.route("/charts/power", methods=["POST"]) From 74ebb2cb22b4713c0d032158ec1feeacc9c7ab2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 23 Feb 2021 11:59:54 +0100 Subject: [PATCH 04/12] Make /charts/power a GET request again --- flexmeasures/ui/views/charts.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/flexmeasures/ui/views/charts.py b/flexmeasures/ui/views/charts.py index f299cc47f..baf4c53e6 100644 --- a/flexmeasures/ui/views/charts.py +++ b/flexmeasures/ui/views/charts.py @@ -26,7 +26,7 @@ v2_0_service_listing["services"].append( { - "name": "POST /charts/power", + "name": "GET /charts/power", "access": ["admin", "Prosumer"], "description": "Get a Bokeh chart for power data to embed in web pages.", }, @@ -46,10 +46,10 @@ class ChartRequestSchema(Schema): forecast_horizon = DurationField(missing="PT6H") # TODO: HorizonField -@flexmeasures_api_v2_0.route("/charts/power", methods=["POST"]) +@flexmeasures_api_v2_0.route("/charts/power", methods=["GET"]) @cross_origin() # origins=["localhost"]) @roles_accepted("admin", "Prosumer") -@use_args(ChartRequestSchema()) +@use_args(ChartRequestSchema(), location="querystring") @as_json def get_power_chart(chart_request): """API endpoint to get a chart for power data which can be embedded in web pages. @@ -85,20 +85,19 @@ def get_power_chart(chart_request): .. sourcecode:: javascript - where `data` contains the chart request parameters (see above). + where `urlData` is a `URLSearchData` object and contains the chart request parameters (see above). :reqheader Authorization: The authentication token :reqheader Content-Type: application/json From 1b847cb7e6c29433f00c85316797e04b42876e77 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Feb 2021 15:12:28 +0100 Subject: [PATCH 05/12] Account for daylight saving time transitions. Test for daylight saving time transitions. --- flexmeasures/api/common/schemas.py | 12 ++++++++++-- flexmeasures/api/common/tests/test_schemas.py | 15 +++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/common/schemas.py b/flexmeasures/api/common/schemas.py index 1412a5790..21e609dd6 100644 --- a/flexmeasures/api/common/schemas.py +++ b/flexmeasures/api/common/schemas.py @@ -4,6 +4,7 @@ from marshmallow import fields, ValidationError import isodate from isodate.isoerror import ISO8601Error +import pandas as pd class DurationField(fields.Str): @@ -43,8 +44,15 @@ def ground_from( converting to a datetime.timedelta is not possible (no obvious number of days). In this case, `_deserialize` returned an `isodate.Duration`. We can derive the timedelta by grounding to an - actual time span, for which we require a start datetime. + actual time span, for which we require a timezone-aware start datetime. """ if isinstance(duration, isodate.Duration) and start: - return (start + duration) - start + years = duration.years + months = duration.months + days = duration.days + seconds = duration.tdelta.seconds + offset = pd.DateOffset( + years=years, months=months, days=days, seconds=seconds + ) + return (pd.Timestamp(start) + offset).to_pydatetime() - start return duration diff --git a/flexmeasures/api/common/tests/test_schemas.py b/flexmeasures/api/common/tests/test_schemas.py index ed953682b..8a239056c 100644 --- a/flexmeasures/api/common/tests/test_schemas.py +++ b/flexmeasures/api/common/tests/test_schemas.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta import pytest +import pytz import isodate from marshmallow import ValidationError @@ -30,6 +31,12 @@ def test_duration_field_straightforward(duration_input, exp_deserialization): ("P1M", isodate.Duration(months=1), timedelta(days=29)), ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)), ("P2D", isodate.Duration(hours=48), timedelta(hours=48)), + # following are calendar periods including a transition to daylight saving time (DST) + ("P2M", isodate.Duration(months=2), timedelta(days=60) - timedelta(hours=1)), + # ("P8W", isodate.Duration(days=7*8), timedelta(weeks=8) - timedelta(hours=1)), + # ("P100D", isodate.Duration(days=100), timedelta(days=100) - timedelta(hours=1)), + # following is a calendar period with transitions to DST and back again + ("P1Y", isodate.Duration(years=1), timedelta(days=366)), ], ) def test_duration_field_nominal_grounded( @@ -38,12 +45,16 @@ def test_duration_field_nominal_grounded( """Nominal durations are tricky: https://en.wikipedia.org/wiki/Talk:ISO_8601/Archive_2#Definition_of_Duration_is_incorrect We want to test if we can ground them as expected. - We use a particular datetime to ground, in a leap year february. + We use a particular datetime to ground, in a leap year February. + For the Europe/Amsterdam timezone, daylight saving time started on March 29th 2020. + # todo: the commented out tests would work if isodate.parse_duration would have the option to stop coercing ISO 8601 days into datetime.timedelta days """ df = DurationField() deser = df.deserialize(duration_input, None, None) assert deser == exp_deserialization - dummy_time = datetime(2020, 2, 22, 18, 7) + dummy_time = pytz.timezone("Europe/Amsterdam").localize( + datetime(2020, 2, 22, 18, 7) + ) grounded = DurationField.ground_from(deser, dummy_time) assert grounded == grounded_timedelta From 6691b43e21a07bc30a63257e90f921784d70de9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 23 Feb 2021 16:55:45 +0100 Subject: [PATCH 06/12] Add DurationValidationError and let the error handler catch it. API error handling of FMValildationErrors now turns them into USEF-compatible responses. --- flexmeasures/api/common/schemas.py | 10 +++- flexmeasures/api/common/tests/test_schemas.py | 5 +- flexmeasures/api/common/utils/args_parsing.py | 49 ++++++++++++------- flexmeasures/api/v1_2/tests/test_api_v1_2.py | 2 +- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/flexmeasures/api/common/schemas.py b/flexmeasures/api/common/schemas.py index 1412a5790..172eeb6e2 100644 --- a/flexmeasures/api/common/schemas.py +++ b/flexmeasures/api/common/schemas.py @@ -1,10 +1,16 @@ from typing import Union, Optional from datetime import datetime, timedelta -from marshmallow import fields, ValidationError +from marshmallow import fields import isodate from isodate.isoerror import ISO8601Error +from flexmeasures.api.common.utils.args_parsing import FMValidationError + + +class DurationValidationError(FMValidationError): + status = "INVALID_PERIOD" # USEF error status + class DurationField(fields.Str): """Field that deserializes to a ISO8601 Duration @@ -22,7 +28,7 @@ def _deserialize( try: return isodate.parse_duration(value) except ISO8601Error as iso_err: - raise ValidationError( + raise DurationValidationError( f"Cannot parse {value} as ISO8601 duration: {iso_err}" ) diff --git a/flexmeasures/api/common/tests/test_schemas.py b/flexmeasures/api/common/tests/test_schemas.py index ed953682b..9f057f81d 100644 --- a/flexmeasures/api/common/tests/test_schemas.py +++ b/flexmeasures/api/common/tests/test_schemas.py @@ -2,9 +2,8 @@ import pytest import isodate -from marshmallow import ValidationError -from flexmeasures.api.common.schemas import DurationField +from flexmeasures.api.common.schemas import DurationField, DurationValidationError @pytest.mark.parametrize( @@ -59,6 +58,6 @@ def test_duration_field_nominal_grounded( ) def test_duration_field_invalid(duration_input, error_msg): df = DurationField() - with pytest.raises(ValidationError) as ve: + with pytest.raises(DurationValidationError) as ve: df.deserialize(duration_input, None, None) assert error_msg in str(ve) diff --git a/flexmeasures/api/common/utils/args_parsing.py b/flexmeasures/api/common/utils/args_parsing.py index 1d868fb88..46d9a0862 100644 --- a/flexmeasures/api/common/utils/args_parsing.py +++ b/flexmeasures/api/common/utils/args_parsing.py @@ -4,37 +4,48 @@ from webargs.flaskparser import parser """ -Utils for argument parsing (we use webargs) +Utils for argument parsing (we use webargs), +including error handling. """ -class FMValidationError(Exception): - """ Custom validation error class """ +@parser.error_handler +def handle_error(error, req, schema, *, error_status_code, error_headers): + """Replacing webargs's error parser, so we can throw custom Exceptions.""" + if error.__class__ == ValidationError: + # re-package all marshmallow's validation errors as our own kind (see below) + raise FMValidationError(message=error.messages) + raise error + + +class FMValidationError(ValidationError): + """ + Custom validation error class. + It differs from the classic validation error by having two + attributes, according to the USEF 2015 reference implementation. + Subclasses of this error might adjust the `status` attribute accordingly. + """ - def __init__(self, messages): - self.result = "Rejected" - self.status = "UNPROCESSABLE_ENTITY" - self.messages = messages + result = "Rejected" + status = "UNPROCESSABLE_ENTITY" -def validation_error_handler(error): - """Handles errors during parsing. Aborts the current HTTP request and - responds with a 422 error. +def validation_error_handler(error: FMValidationError): + """Handles errors during parsing. + Aborts the current HTTP request and responds with a 422 error. + FMValidationError attributes "result" and "status" are packaged in the response. """ status_code = 422 - response = jsonify(error.messages) + response_data = dict(message=error.messages) + if hasattr(error, "result"): + response_data["result"] = error.result + if hasattr(error, "status"): + response_data["status"] = error.status + response = jsonify(response_data) response.status_code = status_code return response -@parser.error_handler -def handle_error(error, req, schema, *, error_status_code, error_headers): - """Replacing webargs's error parser, so we can throw custom Exceptions.""" - if error.__class__ == ValidationError: - raise FMValidationError(messages=error.messages) - raise error - - @parser.location_loader("args_and_json") def load_data(request, schema): """ diff --git a/flexmeasures/api/v1_2/tests/test_api_v1_2.py b/flexmeasures/api/v1_2/tests/test_api_v1_2.py index 1f4da1e32..c439d3c83 100644 --- a/flexmeasures/api/v1_2/tests/test_api_v1_2.py +++ b/flexmeasures/api/v1_2/tests/test_api_v1_2.py @@ -69,7 +69,7 @@ def test_get_device_message_mistyped_duration(client): assert get_device_message_response.status_code == 422 assert ( "Cannot parse PTT6H as ISO8601 duration" - in get_device_message_response.json["args_and_json"]["duration"][0] + in get_device_message_response.json["message"]["args_and_json"]["duration"][0] ) From ec6aa6a37b8a3fd87c55417ecf4e248f0f4c1997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 23 Feb 2021 17:45:38 +0100 Subject: [PATCH 07/12] adapt asset tests, as all our FMValidationErrors now have an extra layer (result, message, status) --- flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py index 222ef094b..9b26a584f 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py @@ -166,7 +166,7 @@ def test_post_an_asset_with_nonexisting_field(client): headers={"content-type": "application/json", "Authorization": auth_token}, ) assert asset_creation.status_code == 422 - assert asset_creation.json["json"]["nnname"][0] == "Unknown field." + assert asset_creation.json["message"]["json"]["nnname"][0] == "Unknown field." def test_posting_multiple_assets(client): @@ -183,7 +183,7 @@ def test_posting_multiple_assets(client): ) print(f"Response: {asset_creation.json}") assert asset_creation.status_code == 422 - assert asset_creation.json["json"]["_schema"][0] == "Invalid input type." + assert asset_creation.json["message"]["json"]["_schema"][0] == "Invalid input type." def test_post_an_asset(client): @@ -234,16 +234,16 @@ def test_post_an_asset_with_invalid_data(client, db): assert ( "Must be greater than or equal to 0" - in post_asset_response.json["json"]["capacity_in_mw"][0] + in post_asset_response.json["message"]["json"]["capacity_in_mw"][0] ) assert ( "greater than or equal to -180 and less than or equal to 180" - in post_asset_response.json["json"]["longitude"][0] + in post_asset_response.json["message"]["json"]["longitude"][0] ) - assert "required field" in post_asset_response.json["json"]["unit"][0] + assert "required field" in post_asset_response.json["message"]["json"]["unit"][0] assert ( "must be equal or higher than the minimum soc" - in post_asset_response.json["json"]["max_soc_in_mwh"] + in post_asset_response.json["message"]["json"]["max_soc_in_mwh"] ) assert Asset.query.filter_by(owner_id=prosumer.id).count() == num_assets_before From bd90b11f57bbdadf2b978f7056ca3af8f5532dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 23 Feb 2021 22:02:43 +0100 Subject: [PATCH 08/12] configure Flask-Cors correctly, actually set our endpoint on the /api/v2_0/ path --- flexmeasures/app.py | 2 ++ flexmeasures/ui/__init__.py | 4 +++- flexmeasures/ui/views/charts.py | 9 ++++----- flexmeasures/utils/config_defaults.py | 8 +++++++- requirements/app.txt | 5 +++-- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 989f1d2ba..e2663c4cc 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -6,6 +6,7 @@ from flask_mail import Mail from flask_sslify import SSLify from flask_json import FlaskJSON +from flask_cors import CORS from redis import Redis from rq import Queue @@ -43,6 +44,7 @@ def create(env=None) -> Flask: app.mail = Mail(app) FlaskJSON(app) + cors = CORS(app) # configure Redis (for redis queue) if app.testing: diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index a7c4e6639..21161f7a5 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -59,7 +59,9 @@ def favicon(): add_jinja_variables(app) # re-register api blueprint so it'll register the chart views (see views.charts) - app.register_blueprint(flexmeasures_api_v2_0, first_registration=False) + app.register_blueprint( + flexmeasures_api_v2_0, url_prefix="/api/v2_0", first_registration=False + ) def register_rq_dashboard(app): diff --git a/flexmeasures/ui/views/charts.py b/flexmeasures/ui/views/charts.py index baf4c53e6..c6ca8074b 100644 --- a/flexmeasures/ui/views/charts.py +++ b/flexmeasures/ui/views/charts.py @@ -1,9 +1,8 @@ from flask_security import roles_accepted +from flask_json import as_json from bokeh.embed import json_item from marshmallow import Schema, fields from webargs.flaskparser import use_args -from flask_cors import cross_origin -from flask_json import as_json from flexmeasures.api.v2_0 import flexmeasures_api as flexmeasures_api_v2_0 from flexmeasures.api.v2_0.routes import v2_0_service_listing @@ -47,7 +46,6 @@ class ChartRequestSchema(Schema): @flexmeasures_api_v2_0.route("/charts/power", methods=["GET"]) -@cross_origin() # origins=["localhost"]) @roles_accepted("admin", "Prosumer") @use_args(ChartRequestSchema(), location="querystring") @as_json @@ -85,9 +83,10 @@ def get_power_chart(chart_request): .. sourcecode:: javascript