Let CLI package use custom Marshmallow Field definitions

documentation/changelog.rst

@@ -13,12 +13,21 @@ New features
 
 Bugfixes
 -----------
-* Prevent logging out user when clearing the session [see `PR #112 <http://www.github.com/SeitaBV/flexmeasures/pull/112>`_]
+
 
 Infrastructure / Support
 ----------------------
 * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 <http://www.github.com/SeitaBV/flexmeasures/pull/108>`_]
+* Let CLI package and plugins use Marshmallow Field definitions [see `PR #125 <http://www.github.com/SeitaBV/flexmeasures/pull/125>`_]
+
+
+v0.4.1 | May XX, 2021
+===========================
 
+Bugfixes
+-----------
+* Prevent logging out user when clearing the session [see `PR #112 <http://www.github.com/SeitaBV/flexmeasures/pull/112>`_]
+* Prevent user type data source to be created without setting a user [see `PR #111 <https://github.com/SeitaBV/flexmeasures/pull/111>`_]
 
 v0.4.0 | April 29, 2021
 ===========================

documentation/dev/plugins.rst

@@ -80,6 +80,45 @@ All else that is needed for this showcase (not shown here) is ``<some_folder>/ou
 
 .. note:: Plugin views can also be added to the FlexMeasures UI menu ― just name them in the config setting :ref:`menu-config`.
 
+Validating data with marshmallow
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+FlexMeasures validates input data using `marshmallow <https://marshmallow.readthedocs.io/>`_.
+Data fields can be made suitable for use in CLI commands through our ``MarshmallowClickMixin``.
+An example:
+
+.. code-block:: python
+
+    from datetime import datetime
+    from typing import Optional
+
+    import click
+    from flexmeasures.data.schemas.times import AwareDateTimeField
+    from flexmeasures.data.schemas.utils import MarshmallowClickMixin
+    from marshmallow import fields
+
+    class StrField(fields.Str, MarshmallowClickMixin):
+        """String field validator usable for UI routes and CLI functions."""
+
+    @click.command("meet")
+    @click.option(
+        "--where",
+        required=True,
+        type=StrField(),  # see above: we just made this field suitable for CLI functions
+        help="(Required) Where we meet",
+    )
+    @click.option(
+        "--when",
+        required=False,
+        type=AwareDateTimeField(format="iso"),  # FlexMeasures already made this field suitable for CLI functions
+        help="[Optional] When we meet (expects timezone-aware ISO 8601 datetime format)",
+    )
+    def schedule_meeting(
+        where: str,
+        when: Optional[datetime] = None,
+    ):
+        print(f"Okay, see you {where} on {when}.")
+
 
 Using other files in your plugin
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

flexmeasures/api/__init__.py

@@ -6,9 +6,9 @@
 from flexmeasures import __version__ as flexmeasures_version
 from flexmeasures.data.models.user import User
 from flexmeasures.api.common.utils.args_parsing import (
-    FMValidationError,
     validation_error_handler,
 )
+from flexmeasures.data.schemas.utils import FMValidationError
 
 # The api blueprint. It is registered with the Flask app (see app.py)
 flexmeasures_api = Blueprint("flexmeasures_api", __name__)

flexmeasures/api/common/utils/args_parsing.py

@@ -1,4 +1,5 @@
 from flask import jsonify
+from flexmeasures.data.schemas.utils import FMValidationError
 from webargs.multidictproxy import MultiDictProxy
 from webargs import ValidationError
 from webargs.flaskparser import parser
@@ -18,18 +19,6 @@ def handle_error(error, req, schema, *, error_status_code, error_headers):
     raise error
 
 
-class FMValidationError(ValidationError):
-    """
-    Custom validation error class.
-    It differs from the classic validation error by having two
-    attributes, according to the USEF 2015 reference implementation.
-    Subclasses of this error might adjust the `status` attribute accordingly.
-    """
-
-    result = "Rejected"
-    status = "UNPROCESSABLE_ENTITY"
-
-
 def validation_error_handler(error: FMValidationError):
     """Handles errors during parsing.
     Aborts the current HTTP request and responds with a 422 error.

flexmeasures/api/common/utils/validators.py

@@ -16,7 +16,7 @@
 
 from webargs.flaskparser import parser
 
-from flexmeasures.api.common.schemas.times import DurationField
+from flexmeasures.data.schemas.times import DurationField
 from flexmeasures.api.common.responses import (  # noqa: F401
     required_info_missing,
     invalid_horizon,

flexmeasures/api/dev/sensors.py

@@ -7,7 +7,7 @@
 from webargs.flaskparser import use_kwargs
 from werkzeug.exceptions import abort
 
-from flexmeasures.api.common.schemas.times import AwareDateTimeField
+from flexmeasures.data.schemas.times import AwareDateTimeField
 from flexmeasures.data.models.time_series import Sensor
 
 

flexmeasures/api/v2_0/implementations/assets.py

@@ -9,7 +9,8 @@
 from marshmallow import fields
 
 from flexmeasures.data.services.resources import get_assets
-from flexmeasures.data.models.assets import Asset as AssetModel, AssetSchema
+from flexmeasures.data.models.assets import Asset as AssetModel
+from flexmeasures.data.schemas.assets import AssetSchema
 from flexmeasures.data.auth_setup import unauthorized_handler
 from flexmeasures.data.config import db
 from flexmeasures.api.common.responses import required_info_missing

flexmeasures/api/v2_0/implementations/users.py

@@ -1,16 +1,15 @@
 from functools import wraps
 
 from flask import current_app, abort
-from marshmallow import ValidationError, validate, validates, fields
+from marshmallow import fields
 from sqlalchemy.exc import IntegrityError
 from webargs.flaskparser import use_args
 from flask_security import current_user
 from flask_security.recoverable import send_reset_password_instructions
 from flask_json import as_json
-from pytz import all_timezones
 
-from flexmeasures.data import ma
 from flexmeasures.data.models.user import User as UserModel
+from flexmeasures.data.schemas.users import UserSchema
 from flexmeasures.data.services.users import (
     get_users,
     set_random_password,
@@ -26,28 +25,6 @@
 Both POST (to create) and DELETE are not accessible via the API, but as CLI functions.
 """
 
-
-class UserSchema(ma.SQLAlchemySchema):
-    """
-    This schema lists fields we support through this API (e.g. no password).
-    """
-
-    class Meta:
-        model = UserModel
-
-    @validates("timezone")
-    def validate_timezone(self, timezone):
-        if timezone not in all_timezones:
-            raise ValidationError(f"Timezone {timezone} doesn't exist.")
-
-    id = ma.auto_field()
-    email = ma.auto_field(required=True, validate=validate.Email)
-    username = ma.auto_field(required=True)
-    active = ma.auto_field()
-    timezone = ma.auto_field()
-    flexmeasures_roles = ma.auto_field()
-
-
 user_schema = UserSchema()
 users_schema = UserSchema(many=True)
 

flexmeasures/data/models/assets.py

@@ -3,13 +3,9 @@
 import isodate
 import timely_beliefs as tb
 from sqlalchemy.orm import Query
-from marshmallow import ValidationError, validate, validates, fields, validates_schema
 
 from flexmeasures.data.config import db
-from flexmeasures.data import ma
-from flexmeasures.data.models.time_series import Sensor, SensorSchemaMixin, TimedValue
-from flexmeasures.data.models.markets import Market
-from flexmeasures.data.models.user import User
+from flexmeasures.data.models.time_series import Sensor, TimedValue
 from flexmeasures.utils.entity_address_utils import build_entity_address
 from flexmeasures.utils.flexmeasures_inflection import humanize, pluralize
 
@@ -182,67 +178,6 @@ def __repr__(self):
         )
 
 
-class AssetSchema(SensorSchemaMixin, ma.SQLAlchemySchema):
-    """
-    Asset schema, with validations.
-    """
-
-    class Meta:
-        model = Asset
-
-    @validates("name")
-    def validate_name(self, name: str):
-        asset = Asset.query.filter(Asset.name == name).one_or_none()
-        if asset:
-            raise ValidationError(f"An asset with the name {name} already exists.")
-
-    @validates("owner_id")
-    def validate_owner(self, owner_id: int):
-        owner = User.query.get(owner_id)
-        if not owner:
-            raise ValidationError(f"Owner with id {owner_id} doesn't exist.")
-        if "Prosumer" not in owner.flexmeasures_roles:
-            raise ValidationError(
-                "Asset owner must have role 'Prosumer'."
-                f" User {owner_id} has roles {[r.name for r in owner.flexmeasures_roles]}."
-            )
-
-    @validates("market_id")
-    def validate_market(self, market_id: int):
-        market = Market.query.get(market_id)
-        if not market:
-            raise ValidationError(f"Market with id {market_id} doesn't exist.")
-
-    @validates("asset_type_name")
-    def validate_asset_type(self, asset_type_name: str):
-        asset_type = AssetType.query.get(asset_type_name)
-        if not asset_type:
-            raise ValidationError(f"Asset type {asset_type_name} doesn't exist.")
-
-    @validates_schema(skip_on_field_errors=False)
-    def validate_soc_constraints(self, data, **kwargs):
-        if "max_soc_in_mwh" in data and "min_soc_in_mwh" in data:
-            if data["max_soc_in_mwh"] < data["min_soc_in_mwh"]:
-                errors = {
-                    "max_soc_in_mwh": "This value must be equal or higher than the minimum soc."
-                }
-                raise ValidationError(errors)
-
-    id = ma.auto_field()
-    display_name = fields.Str(validate=validate.Length(min=4))
-    capacity_in_mw = fields.Float(required=True, validate=validate.Range(min=0.0001))
-    min_soc_in_mwh = fields.Float(validate=validate.Range(min=0))
-    max_soc_in_mwh = fields.Float(validate=validate.Range(min=0))
-    soc_in_mwh = ma.auto_field()
-    soc_datetime = ma.auto_field()
-    soc_udi_event_id = ma.auto_field()
-    latitude = fields.Float(required=True, validate=validate.Range(min=-90, max=90))
-    longitude = fields.Float(required=True, validate=validate.Range(min=-180, max=180))
-    asset_type_name = ma.auto_field(required=True)
-    owner_id = ma.auto_field(required=True)
-    market_id = ma.auto_field(required=True)
-
-
 def assets_share_location(assets: List[Asset]) -> bool:
     """
     Return True if all assets in this list are located on the same spot.

flexmeasures/data/models/time_series.py

@@ -6,10 +6,8 @@
 from sqlalchemy.orm import Query, Session
 import timely_beliefs as tb
 import timely_beliefs.utils as tb_utils
-from marshmallow import Schema, fields
 
 from flexmeasures.data.config import db
-from flexmeasures.data import ma
 from flexmeasures.data.queries.utils import (
     add_belief_timing_filter,
     add_user_source_filter,
@@ -240,35 +238,6 @@ def __repr__(self) -> str:
         return tb.TimedBelief.__repr__(self)
 
 
-class SensorSchemaMixin(Schema):
-    """
-    Base sensor schema.
-
-    Here we include all fields which are implemented by timely_beliefs.SensorDBMixin
-    All classes inheriting from timely beliefs sensor don't need to repeat these.
-    In a while, this schema can represent our unified Sensor class.
-
-    When subclassing, also subclass from `ma.SQLAlchemySchema` and add your own DB model class, e.g.:
-
-        class Meta:
-            model = Asset
-    """
-
-    name = ma.auto_field(required=True)
-    unit = ma.auto_field(required=True)
-    timezone = ma.auto_field()
-    event_resolution = fields.TimeDelta(required=True, precision="minutes")
-
-
-class SensorSchema(SensorSchemaMixin, ma.SQLAlchemySchema):
-    """
-    Sensor schema, with validations.
-    """
-
-    class Meta:
-        model = Sensor
-
-
 class TimedValue(object):
     """
     A mixin of all tables that store time series data, either forecasts or measurements.

flexmeasures/data/models/weather.py

@@ -6,12 +6,10 @@
 from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
 from sqlalchemy.sql.expression import func
 from sqlalchemy.schema import UniqueConstraint
-from marshmallow import ValidationError, validates, validate, fields
 
 from flexmeasures.data.config import db
 
-from flexmeasures.data import ma
-from flexmeasures.data.models.time_series import Sensor, SensorSchemaMixin, TimedValue
+from flexmeasures.data.models.time_series import Sensor, TimedValue
 from flexmeasures.utils.geo_utils import parse_lat_lng
 from flexmeasures.utils.flexmeasures_inflection import humanize
 
@@ -171,37 +169,6 @@ def to_dict(self) -> Dict[str, str]:
         return dict(name=self.name, sensor_type=self.weather_sensor_type_name)
 
 
-class WeatherSensorSchema(SensorSchemaMixin, ma.SQLAlchemySchema):
-    """
-    WeatherSensor schema, with validations.
-    """
-
-    class Meta:
-        model = WeatherSensor
-
-    @validates("name")
-    def validate_name(self, name: str):
-        sensor = WeatherSensor.query.filter(
-            WeatherSensor.name == name.lower()
-        ).one_or_none()
-        if sensor:
-            raise ValidationError(
-                f"A weather sensor with the name {name} already exists."
-            )
-
-    @validates("weather_sensor_type_name")
-    def validate_weather_sensor_type(self, weather_sensor_type_name: str):
-        weather_sensor_type = WeatherSensorType.query.get(weather_sensor_type_name)
-        if not weather_sensor_type:
-            raise ValidationError(
-                f"Weather sensor type {weather_sensor_type_name} doesn't exist."
-            )
-
-    weather_sensor_type_name = ma.auto_field(required=True)
-    latitude = fields.Float(required=True, validate=validate.Range(min=-90, max=90))
-    longitude = fields.Float(required=True, validate=validate.Range(min=-180, max=180))
-
-
 class Weather(TimedValue, db.Model):
     """
     All weather measurements are stored in one slim table.