first working version of a charts/power view;

flexmeasures/api/common/schemas/tests/test_times.py

@@ -0,0 +1,74 @@
+from datetime import datetime, timedelta
+
+import pytest
+import pytz
+import isodate
+
+from flexmeasures.api.common.schemas.times import DurationField, DurationValidationError
+
+
+@pytest.mark.parametrize(
+    "duration_input,exp_deserialization",
+    [
+        ("PT1H", timedelta(hours=1)),
+        ("PT6M", timedelta(minutes=6)),
+        ("PT6H", timedelta(hours=6)),
+        ("P2DT1H", timedelta(hours=49)),
+    ],
+)
+def test_duration_field_straightforward(duration_input, exp_deserialization):
+    """Testing straightforward cases"""
+    df = DurationField()
+    deser = df.deserialize(duration_input, None, None)
+    assert deser == exp_deserialization
+    assert df.serialize("duration", {"duration": deser}) == duration_input
+
+
+@pytest.mark.parametrize(
+    "duration_input,exp_deserialization,grounded_timedelta",
+    [
+        ("P1M", isodate.Duration(months=1), timedelta(days=29)),
+        ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)),
+        ("P2D", isodate.Duration(hours=48), timedelta(hours=48)),
+        # following are calendar periods including a transition to daylight saving time (DST)
+        ("P2M", isodate.Duration(months=2), timedelta(days=60) - timedelta(hours=1)),
+        # ("P8W", isodate.Duration(days=7*8), timedelta(weeks=8) - timedelta(hours=1)),
+        # ("P100D", isodate.Duration(days=100), timedelta(days=100) - timedelta(hours=1)),
+        # following is a calendar period with transitions to DST and back again
+        ("P1Y", isodate.Duration(years=1), timedelta(days=366)),
+    ],
+)
+def test_duration_field_nominal_grounded(
+    duration_input, exp_deserialization, grounded_timedelta
+):
+    """Nominal durations are tricky:
+    https://en.wikipedia.org/wiki/Talk:ISO_8601/Archive_2#Definition_of_Duration_is_incorrect
+    We want to test if we can ground them as expected.
+    We use a particular datetime to ground, in a leap year February.
+    For the Europe/Amsterdam timezone, daylight saving time started on March 29th 2020.
+    # todo: the commented out tests would work if isodate.parse_duration would have the option to stop coercing ISO 8601 days into datetime.timedelta days
+    """
+    df = DurationField()
+    deser = df.deserialize(duration_input, None, None)
+    assert deser == exp_deserialization
+    dummy_time = pytz.timezone("Europe/Amsterdam").localize(
+        datetime(2020, 2, 22, 18, 7)
+    )
+    grounded = DurationField.ground_from(deser, dummy_time)
+    assert grounded == grounded_timedelta
+
+
+@pytest.mark.parametrize(
+    "duration_input,error_msg",
+    [
+        ("", "Unable to parse duration string"),
+        ("1H", "Unable to parse duration string"),
+        ("PP1M", "time designator 'T' missing"),
+        ("PT2D", "Unrecognised ISO 8601 date format"),
+    ],
+)
+def test_duration_field_invalid(duration_input, error_msg):
+    df = DurationField()
+    with pytest.raises(DurationValidationError) as ve:
+        df.deserialize(duration_input, None, None)
+    assert error_msg in str(ve)

flexmeasures/api/common/schemas/times.py

@@ -0,0 +1,64 @@
+from typing import Union, Optional
+from datetime import datetime, timedelta
+
+from marshmallow import fields
+import isodate
+from isodate.isoerror import ISO8601Error
+import pandas as pd
+
+from flexmeasures.api.common.utils.args_parsing import FMValidationError
+
+
+class DurationValidationError(FMValidationError):
+    status = "INVALID_PERIOD"  # USEF error status
+
+
+class DurationField(fields.Str):
+    """Field that deserializes to a ISO8601 Duration
+    and serializes back to a string."""
+
+    def _deserialize(
+        self, value, attr, obj, **kwargs
+    ) -> Union[timedelta, isodate.Duration]:
+        """
+        Use the isodate library to turn an ISO8601 string into a timedelta.
+        For some non-obvious cases, it will become an isodate.Duration, see
+        ground_from for more.
+        This method throws a ValidationError if the string is not ISO norm.
+        """
+        try:
+            return isodate.parse_duration(value)
+        except ISO8601Error as iso_err:
+            raise DurationValidationError(
+                f"Cannot parse {value} as ISO8601 duration: {iso_err}"
+            )
+
+    def _serialize(self, value, attr, data, **kwargs):
+        """
+        An implementation of _serialize.
+        It is not guaranteed to return the same string as was input,
+        if ground_from has been used!
+        """
+        return isodate.strftime(value, "P%P")
+
+    @staticmethod
+    def ground_from(
+        duration: Union[timedelta, isodate.Duration], start: Optional[datetime]
+    ) -> timedelta:
+        """
+        For some valid duration strings (such as "P1M", a month),
+        converting to a datetime.timedelta is not possible (no obvious
+        number of days). In this case, `_deserialize` returned an
+        `isodate.Duration`. We can derive the timedelta by grounding to an
+        actual time span, for which we require a timezone-aware start datetime.
+        """
+        if isinstance(duration, isodate.Duration) and start:
+            years = duration.years
+            months = duration.months
+            days = duration.days
+            seconds = duration.tdelta.seconds
+            offset = pd.DateOffset(
+                years=years, months=months, days=days, seconds=seconds
+            )
+            return (pd.Timestamp(start) + offset).to_pydatetime() - start
+        return duration

flexmeasures/api/common/utils/args_parsing.py

@@ -4,37 +4,48 @@
 from webargs.flaskparser import parser
 
 """
-Utils for argument parsing (we use webargs)
+Utils for argument parsing (we use webargs),
+including error handling.
 """
 
 
-class FMValidationError(Exception):
-    """ Custom validation error class """
+@parser.error_handler
+def handle_error(error, req, schema, *, error_status_code, error_headers):
+    """Replacing webargs's error parser, so we can throw custom Exceptions."""
+    if error.__class__ == ValidationError:
+        # re-package all marshmallow's validation errors as our own kind (see below)
+        raise FMValidationError(message=error.messages)
+    raise error
+
+
+class FMValidationError(ValidationError):
+    """
+    Custom validation error class.
+    It differs from the classic validation error by having two
+    attributes, according to the USEF 2015 reference implementation.
+    Subclasses of this error might adjust the `status` attribute accordingly.
+    """
 
-    def __init__(self, messages):
-        self.result = "Rejected"
-        self.status = "UNPROCESSABLE_ENTITY"
-        self.messages = messages
+    result = "Rejected"
+    status = "UNPROCESSABLE_ENTITY"
 
 
-def validation_error_handler(error):
-    """Handles errors during parsing. Aborts the current HTTP request and
-    responds with a 422 error.
+def validation_error_handler(error: FMValidationError):
+    """Handles errors during parsing.
+    Aborts the current HTTP request and responds with a 422 error.
+    FMValidationError attributes "result" and "status" are packaged in the response.
     """
     status_code = 422
-    response = jsonify(error.messages)
+    response_data = dict(message=error.messages)
+    if hasattr(error, "result"):
+        response_data["result"] = error.result
+    if hasattr(error, "status"):
+        response_data["status"] = error.status
+    response = jsonify(response_data)
     response.status_code = status_code
     return response
 
 
-@parser.error_handler
-def handle_error(error, req, schema, *, error_status_code, error_headers):
-    """Replacing webargs's error parser, so we can throw custom Exceptions."""
-    if error.__class__ == ValidationError:
-        raise FMValidationError(messages=error.messages)
-    raise error
-
-
 @parser.location_loader("args_and_json")
 def load_data(request, schema):
     """

flexmeasures/api/common/utils/validators.py

@@ -14,9 +14,9 @@
 from flask_security import current_user
 import marshmallow
 
-from webargs import fields
 from webargs.flaskparser import parser
 
+from flexmeasures.api.common.schemas.times import DurationField
 from flexmeasures.api.common.responses import (  # noqa: F401
     required_info_missing,
     invalid_horizon,
@@ -134,30 +134,19 @@ def parse_duration(
     Parses the 'duration' string into a Duration object.
     If needed, try deriving the timedelta from the actual time span (e.g. in case duration is 1 year).
     If the string is not a valid ISO 8601 time interval, return None.
+
+    TODO: Deprecate for DurationField.
     """
     try:
         duration = isodate.parse_duration(duration_str)
-        if not isinstance(duration, timedelta):
-            if start:
-                return (start + duration) - start
-            return duration  # valid duration, but not a timedelta (e.g. "P1Y" could be leap year)
-        else:
-            return isodate.parse_duration(duration_str)
+        if not isinstance(duration, timedelta) and start:
+            return (start + duration) - start
+        # if not a timedelta, then it's a valid duration (e.g. "P1Y" could be leap year)
+        return duration
     except (ISO8601Error, AttributeError):
         return None
 
 
-def validate_duration_field(duration_str):
-    """Validate a marshmallow ISO8601 duration field,
-    throw marshmallow validation error if it cannot be parsed."""
-    try:
-        isodate.parse_duration(duration_str)
-    except ISO8601Error as iso_err:
-        raise marshmallow.ValidationError(
-            f"Cannot parse {duration_str} as ISO8601 duration: {iso_err}"
-        )
-
-
 def parse_isodate_str(start: str) -> Union[datetime, None]:
     """
     Validates whether the string 'start' is a valid ISO 8601 datetime.
@@ -203,18 +192,18 @@ def wrapper(fn):
         @as_json
         def decorated_service(*args, **kwargs):
             duration_arg = parser.parse(
-                {"duration": fields.Str(validate=validate_duration_field)},
+                {"duration": DurationField()},
                 request,
                 location="args_and_json",
                 unknown=marshmallow.EXCLUDE,
             )
-
             if "duration" in duration_arg:
-                duration = parse_duration(
-                    duration_arg["duration"],
+                duration = duration_arg["duration"]
+                duration = DurationField.ground_from(
+                    duration,
                     kwargs.get("start", kwargs.get("datetime", None)),
                 )
-                if not duration:
+                if not duration:  # TODO: deprecate
                     extra_info = "Cannot parse 'duration' value."
                     current_app.logger.warning(extra_info)
                     return invalid_period(extra_info)

flexmeasures/api/v1_2/tests/test_api_v1_2.py

@@ -69,7 +69,7 @@ def test_get_device_message_mistyped_duration(client):
     assert get_device_message_response.status_code == 422
     assert (
         "Cannot parse PTT6H as ISO8601 duration"
-        in get_device_message_response.json["args_and_json"]["duration"][0]
+        in get_device_message_response.json["message"]["args_and_json"]["duration"][0]
     )
 
 

flexmeasures/api/v2_0/tests/test_api_v2_0_assets.py

@@ -166,7 +166,7 @@ def test_post_an_asset_with_nonexisting_field(client):
         headers={"content-type": "application/json", "Authorization": auth_token},
     )
     assert asset_creation.status_code == 422
-    assert asset_creation.json["json"]["nnname"][0] == "Unknown field."
+    assert asset_creation.json["message"]["json"]["nnname"][0] == "Unknown field."
 
 
 def test_posting_multiple_assets(client):
@@ -183,7 +183,7 @@ def test_posting_multiple_assets(client):
     )
     print(f"Response: {asset_creation.json}")
     assert asset_creation.status_code == 422
-    assert asset_creation.json["json"]["_schema"][0] == "Invalid input type."
+    assert asset_creation.json["message"]["json"]["_schema"][0] == "Invalid input type."
 
 
 def test_post_an_asset(client):
@@ -234,16 +234,16 @@ def test_post_an_asset_with_invalid_data(client, db):
 
     assert (
         "Must be greater than or equal to 0"
-        in post_asset_response.json["json"]["capacity_in_mw"][0]
+        in post_asset_response.json["message"]["json"]["capacity_in_mw"][0]
     )
     assert (
         "greater than or equal to -180 and less than or equal to 180"
-        in post_asset_response.json["json"]["longitude"][0]
+        in post_asset_response.json["message"]["json"]["longitude"][0]
     )
-    assert "required field" in post_asset_response.json["json"]["unit"][0]
+    assert "required field" in post_asset_response.json["message"]["json"]["unit"][0]
     assert (
         "must be equal or higher than the minimum soc"
-        in post_asset_response.json["json"]["max_soc_in_mwh"]
+        in post_asset_response.json["message"]["json"]["max_soc_in_mwh"]
     )
 
     assert Asset.query.filter_by(owner_id=prosumer.id).count() == num_assets_before

flexmeasures/app.py

@@ -6,6 +6,7 @@
 from flask_mail import Mail
 from flask_sslify import SSLify
 from flask_json import FlaskJSON
+from flask_cors import CORS
 
 from redis import Redis
 from rq import Queue
@@ -43,6 +44,7 @@ def create(env=None) -> Flask:
 
     app.mail = Mail(app)
     FlaskJSON(app)
+    cors = CORS(app)
 
     # configure Redis (for redis queue)
     if app.testing:
@@ -80,18 +82,18 @@ def create(env=None) -> Flask:
 
     register_db_at(app)
 
-    # Register the UI
-
-    from flexmeasures.ui import register_at as register_ui_at
-
-    register_ui_at(app)
-
     # Register the API
 
     from flexmeasures.api import register_at as register_api_at
 
     register_api_at(app)
 
+    # Register the UI
+
+    from flexmeasures.ui import register_at as register_ui_at
+
+    register_ui_at(app)
+
     # Profile endpoints (if needed, e.g. during development)
     @app.before_request
     def before_request():

flexmeasures/ui/__init__.py

@@ -15,6 +15,7 @@
     localized_datetime_str,
     naturalized_datetime_str,
 )
+from flexmeasures.api.v2_0 import flexmeasures_api as flexmeasures_api_v2_0
 
 # The ui blueprint. It is registered with the Flask app (see app.py)
 flexmeasures_ui = Blueprint(
@@ -57,6 +58,11 @@ def favicon():
     add_jinja_filters(app)
     add_jinja_variables(app)
 
+    # re-register api blueprint so it'll register the chart views (the ones in views.charts)
+    app.register_blueprint(
+        flexmeasures_api_v2_0, url_prefix="/api/v2_0", first_registration=False
+    )
+
 
 def register_rq_dashboard(app):
     app.config.update(