Issue 14 user management via api (#25)

-  open up user data via the API (no creation and deletion though)
-  enable customisation of password reset emails
-  some updates to the asset API with learning from this PR, e.g. use webargs/marshmallow
-  show how we can use webargs/marshmallow for the older parts of the API, which will become a new PR


Co-authored-by: F.N. Claessen <felix@seita.nl>
Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com>

flexmeasures/api/Readme.md

@@ -56,6 +56,12 @@ Utility functions that are commonly shared between endpoint implementations of d
 where we distinguish between response decorators, request validators and other utils.
 
 
+UI Crud
+-------
+
+In `ui/crud`, we support FlexMeasures' in-built UI with Flask endpoints, which then talk to our internal API.
+The routes used there point to an API version. You should consider updating them to point to your new version.
+
 Testing
 -------
 

flexmeasures/api/__init__.py

@@ -5,6 +5,10 @@
 from flask_login import current_user
 
 from flexmeasures.data.models.user import User
+from flexmeasures.api.common.utils.args_parsing import (
+    FMValidationError,
+    validation_error_handler,
+)
 
 # The api blueprint. It is registered with the Flask app (see app.py)
 flexmeasures_api = Blueprint("flexmeasures_api", __name__)
@@ -80,6 +84,9 @@ def register_at(app: Flask):
     global ma
     ma.init_app(app)
 
+    # handle API specific errors
+    app.register_error_handler(FMValidationError, validation_error_handler)
+
     app.register_blueprint(
         flexmeasures_api, url_prefix="/api"
     )  # now registering the blueprint will affect all endpoints

flexmeasures/api/common/utils/api_utils.py

@@ -66,6 +66,7 @@ def parse_as_list(
     return connections
 
 
+# TODO: we should be using webargs to get data from a request, it's more descriptive and has error handling
 def get_form_from_request(_request) -> Union[dict, None]:
     if _request.method == "GET":
         d = _request.args.to_dict(

flexmeasures/api/common/utils/args_parsing.py

@@ -0,0 +1,47 @@
+from flask import jsonify
+from webargs.multidictproxy import MultiDictProxy
+from webargs import ValidationError
+from webargs.flaskparser import parser
+
+"""
+Utils for argument parsing (we use webargs)
+"""
+
+
+class FMValidationError(Exception):
+    """ Custom validation error class """
+
+    def __init__(self, messages):
+        self.result = "Rejected"
+        self.status = "UNPROCESSABLE_ENTITY"
+        self.messages = messages
+
+
+def validation_error_handler(error):
+    """Handles errors during parsing. Aborts the current HTTP request and
+    responds with a 422 error.
+    """
+    status_code = 422
+    response = jsonify(error.messages)
+    response.status_code = status_code
+    return response
+
+
+@parser.error_handler
+def handle_error(error, req, schema, *, error_status_code, error_headers):
+    """Replacing webargs's error parser, so we can throw custom Exceptions."""
+    if error.__class__ == ValidationError:
+        raise FMValidationError(messages=error.messages)
+    raise error
+
+
+@parser.location_loader("args_and_json")
+def load_data(request, schema):
+    """
+    We allow parameters to come from either GET args or POST JSON,
+    as validators can be attached to either.
+    """
+    newdata = request.args.copy()
+    if request.mimetype == "application/json" and request.method == "POST":
+        newdata.update(request.get_json())
+    return MultiDictProxy(newdata, schema)

flexmeasures/api/common/utils/validators.py

@@ -12,6 +12,10 @@
 from flask_json import as_json
 from flask_principal import Permission, RoleNeed
 from flask_security import current_user
+import marshmallow
+
+from webargs import fields
+from webargs.flaskparser import parser
 
 from flexmeasures.api.common.responses import (  # noqa: F401
     required_info_missing,
@@ -87,7 +91,7 @@ def include_current_user_source_id(source_ids: List[int]) -> List[int]:
     return list(set(source_ids))  # only unique source ids
 
 
-def validate_horizon(horizon_str: str) -> Tuple[Optional[timedelta], bool]:
+def parse_horizon(horizon_str: str) -> Tuple[Optional[timedelta], bool]:
     """
     Validates whether a horizon string represents a valid ISO 8601 (repeating) time interval.
 
@@ -123,12 +127,13 @@ def validate_horizon(horizon_str: str) -> Tuple[Optional[timedelta], bool]:
     return horizon, is_repetition
 
 
-def validate_duration(
+def parse_duration(
     duration_str: str, start: Optional[datetime] = None
 ) -> Union[timedelta, isodate.Duration, None]:
     """
-    Validates whether the string 'duration' is a valid ISO 8601 time interval.
+    Parses the 'duration' string into a Duration object.
     If needed, try deriving the timedelta from the actual time span (e.g. in case duration is 1 year).
+    If the string is not a valid ISO 8601 time interval, return None.
     """
     try:
         duration = isodate.parse_duration(duration_str)
@@ -142,6 +147,17 @@ def validate_duration(
         return None
 
 
+def validate_duration_field(duration_str):
+    """Validate a marshmallow ISO8601 duration field,
+    throw marshmallow validation error if it cannot be parsed."""
+    try:
+        isodate.parse_duration(duration_str)
+    except ISO8601Error as iso_err:
+        raise marshmallow.ValidationError(
+            f"Cannot parse {duration_str} as ISO8601 duration: {iso_err}"
+        )
+
+
 def parse_isodate_str(start: str) -> Union[datetime, None]:
     """
     Validates whether the string 'start' is a valid ISO 8601 datetime.
@@ -186,16 +202,17 @@ def wrapper(fn):
         @wraps(fn)
         @as_json
         def decorated_service(*args, **kwargs):
-            form = get_form_from_request(request)
-            if form is None:
-                current_app.logger.warning(
-                    "Unsupported request method for unpacking 'duration' from request."
-                )
-                return invalid_method(request.method)
+            duration_arg = parser.parse(
+                {"duration": fields.Str(validate=validate_duration_field)},
+                request,
+                location="args_and_json",
+                unknown=marshmallow.EXCLUDE,
+            )
 
-            if "duration" in form:
-                duration = validate_duration(
-                    form["duration"], kwargs.get("start", kwargs.get("datetime", None))
+            if "duration" in duration_arg:
+                duration = parse_duration(
+                    duration_arg["duration"],
+                    kwargs.get("start", kwargs.get("datetime", None)),
                 )
                 if not duration:
                     extra_info = "Cannot parse 'duration' value."
@@ -316,7 +333,7 @@ def decorated_service(*args, **kwargs):
                 prior = parse_isodate_str(form["prior"])
                 if ex_post is True:
                     start = parse_isodate_str(form["start"])
-                    duration = validate_duration(form["duration"], start)
+                    duration = parse_duration(form["duration"], start)
                     # todo: validate start and duration (refactor already duplicate code from period_required and optional_horizon_accepted)
                     knowledge_time = (
                         start + duration
@@ -380,7 +397,7 @@ def decorated_service(*args, **kwargs):
 
             rolling = True
             if "horizon" in form:
-                horizon, rolling = validate_horizon(form["horizon"])
+                horizon, rolling = parse_horizon(form["horizon"])
                 if horizon is None:
                     current_app.logger.warning("Cannot parse 'horizon' value")
                     return invalid_horizon()
@@ -392,7 +409,7 @@ def decorated_service(*args, **kwargs):
                 # A missing horizon is only accepted if the server can infer it
                 if "start" in form and "duration" in form:
                     start = parse_isodate_str(form["start"])
-                    duration = validate_duration(form["duration"], start)
+                    duration = parse_duration(form["duration"], start)
                     if not start:
                         extra_info = "Cannot parse 'start' value."
                         current_app.logger.warning(extra_info)
@@ -507,7 +524,7 @@ def wrapper(*args, **kwargs):
             return invalid_period()
         kwargs["start"] = start
         if "duration" in form:
-            duration = validate_duration(form["duration"], start)
+            duration = parse_duration(form["duration"], start)
             if not duration:
                 current_app.logger.warning("Cannot parse 'duration' value")
                 return invalid_period()
@@ -865,7 +882,7 @@ def decorated_service(*args, **kwargs):
                 return invalid_method(request.method)
 
             if "resolution" in form and form["resolution"]:
-                ds_resolution = validate_duration(form["resolution"])
+                ds_resolution = parse_duration(form["resolution"])
                 if ds_resolution is None:
                     return invalid_resolution_str(form["resolution"])
                 # Check if the resolution can be applied to all assets (if it is a multiple

flexmeasures/api/v1_2/routes.py

@@ -89,6 +89,7 @@ def get_device_message():
     :status 401: UNAUTHORIZED
     :status 403: INVALID_SENDER
     :status 405: INVALID_METHOD
+    :status 422: UNPROCESSABLE_ENTITY
     """
     return v1_2_implementations.get_device_message_response()
 

flexmeasures/api/v1_2/tests/test_api_v1_2.py

@@ -34,6 +34,8 @@ def test_get_device_message(client, message):
         query_string=message,
         headers={"content-type": "application/json", "Authorization": auth_token},
     )
+    print("Server responded with:\n%s" % get_device_message_response_short.json)
+    assert get_device_message_response_short.status_code == 200
     assert (
         get_device_message_response_short.json["values"]
         == get_device_message_response.json["values"][0:24]
@@ -52,6 +54,25 @@ def test_get_device_message(client, message):
     )
 
 
+def test_get_device_message_mistyped_duration(client):
+    auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest")
+    message = message_for_get_device_message()
+    asset = Asset.query.filter(Asset.name == "Test battery").one_or_none()
+    message["event"] = message["event"] % (asset.owner_id, asset.id)
+    message["duration"] = "PTT6H"
+    get_device_message_response = client.get(
+        url_for("flexmeasures_api_v1_2.get_device_message"),
+        query_string=message,
+        headers={"content-type": "application/json", "Authorization": auth_token},
+    )
+    print("Server responded with:\n%s" % get_device_message_response.json)
+    assert get_device_message_response.status_code == 422
+    assert (
+        "Cannot parse PTT6H as ISO8601 duration"
+        in get_device_message_response.json["args_and_json"]["duration"][0]
+    )
+
+
 @pytest.mark.parametrize("message", [message_for_get_device_message(wrong_id=True)])
 def test_get_device_message_wrong_event_id(client, message):
     asset = Asset.query.filter(Asset.name == "Test battery").one_or_none()

flexmeasures/api/v1_3/routes.py

@@ -70,6 +70,7 @@ def get_device_message():
     :status 401: UNAUTHORIZED
     :status 403: INVALID_SENDER
     :status 405: INVALID_METHOD
+    :status 422: UNPROCESSABLE_ENTITY
     """
     return v1_3_implementations.get_device_message_response()