Issue 6 entity address scheme improvements 2

flexmeasures/api/common/schemas/sensors.py

@@ -0,0 +1,135 @@
+from typing import Union
+
+from marshmallow import fields
+
+from flexmeasures.api import FMValidationError
+from flexmeasures.api.common.utils.api_utils import get_weather_sensor_by
+from flexmeasures.utils.entity_address_utils import (
+    parse_entity_address,
+    EntityAddressException,
+)
+from flexmeasures.data.models.assets import Asset
+from flexmeasures.data.models.markets import Market
+from flexmeasures.data.models.weather import WeatherSensor
+from flexmeasures.data.models.time_series import Sensor
+
+
+class EntityAddressValidationError(FMValidationError):
+    status = "INVALID_DOMAIN"  # USEF error status
+
+
+class SensorField(fields.Str):
+    """Field that deserializes to a Sensor, Asset, Market or WeatherSensor
+    and serializes back to an entity address (string)."""
+
+    # todo: when Actuators also get an entity address, refactor this class to EntityField,
+    #       where an Entity represents anything with an entity address: we currently foresee Sensors and Actuators
+
+    def __init__(
+        self,
+        entity_type: str,
+        fm_scheme: str,
+        *args,
+        **kwargs,
+    ):
+        """
+        :param entity_type: "sensor", "connection", "market" or "weather_sensor"
+        :param fm_scheme:   "fm0" or "fm1"
+        """
+        self.entity_type = entity_type
+        self.fm_scheme = fm_scheme
+        super().__init__(*args, **kwargs)
+
+    def _deserialize(  # noqa: C901 todo: the noqa can probably be removed after refactoring Asset/Market/WeatherSensor to Sensor
+        self, value, attr, obj, **kwargs
+    ) -> Union[Sensor, Asset, Market, WeatherSensor]:
+        """Deserialize to a Sensor, Asset, Market or WeatherSensor."""
+        # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case
+        try:
+            ea = parse_entity_address(value, self.entity_type, self.fm_scheme)
+            if self.fm_scheme == "fm0":
+                if self.entity_type == "connection":
+                    asset = Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none()
+                    if asset is not None:
+                        return asset
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Asset with entity address {value} doesn't exist."
+                        )
+                elif self.entity_type == "market":
+                    market = Market.query.filter(
+                        Market.name == ea["market_name"]
+                    ).one_or_none()
+                    if market is not None:
+                        return market
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Market with entity address {value} doesn't exist."
+                        )
+                elif self.entity_type == "weather_sensor":
+                    weather_sensor = get_weather_sensor_by(
+                        ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"]
+                    )
+                    if weather_sensor is not None and isinstance(
+                        weather_sensor, WeatherSensor
+                    ):
+                        return weather_sensor
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Weather sensor with entity address {value} doesn't exist."
+                        )
+            else:
+                if self.entity_type == "sensor":
+                    sensor = Sensor.query.filter(
+                        Sensor.id == ea["sensor_id"]
+                    ).one_or_none()
+                    if sensor is not None:
+                        return sensor
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Sensor with entity address {value} doesn't exist."
+                        )
+                elif self.entity_type == "connection":
+                    asset = Asset.query.filter(
+                        Asset.id == ea["sensor_id"]
+                    ).one_or_none()
+                    if asset is not None:
+                        return asset
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Asset with entity address {value} doesn't exist."
+                        )
+                elif self.entity_type == "market":
+                    market = Market.query.filter(
+                        Market.id == ea["sensor_id"]
+                    ).one_or_none()
+                    if market is not None:
+                        return market
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Market with entity address {value} doesn't exist."
+                        )
+                elif self.entity_type == "weather_sensor":
+                    weather_sensor = WeatherSensor.query.filter(
+                        WeatherSensor.id == ea["sensor_id"]
+                    ).one_or_none()
+                    if weather_sensor is not None and isinstance(
+                        weather_sensor, WeatherSensor
+                    ):
+                        return weather_sensor
+                    else:
+                        raise EntityAddressValidationError(
+                            f"Weather sensor with entity address {value} doesn't exist."
+                        )
+        except EntityAddressException as eae:
+            raise EntityAddressValidationError(str(eae))
+        return NotImplemented
+
+    def _serialize(
+        self, value: Union[Sensor, Asset, Market, WeatherSensor], attr, data, **kwargs
+    ):
+        """Serialize to an entity address."""
+        if self.fm_scheme == "fm0":
+            return value.entity_address_fm0
+        else:
+            return value.entity_address

flexmeasures/api/common/schemas/tests/test_sensors.py

@@ -0,0 +1,97 @@
+import pytest
+
+from flexmeasures.api.common.schemas.sensors import (
+    SensorField,
+    EntityAddressValidationError,
+)
+from flexmeasures.utils.entity_address_utils import build_entity_address
+
+
+@pytest.mark.parametrize(
+    "entity_address, entity_type, fm_scheme, exp_deserialization_name",
+    [
+        (
+            build_entity_address(dict(sensor_id=9), "sensor"),
+            "sensor",
+            "fm1",
+            "my daughter's height",
+        ),
+        (
+            build_entity_address(
+                dict(market_name="epex_da"), "market", fm_scheme="fm0"
+            ),
+            "market",
+            "fm0",
+            "epex_da",
+        ),
+        (
+            build_entity_address(
+                dict(owner_id=1, asset_id=3), "connection", fm_scheme="fm0"
+            ),
+            "connection",
+            "fm0",
+            "Test battery with no known prices",
+        ),
+        (
+            build_entity_address(
+                dict(
+                    weather_sensor_type_name="temperature",
+                    latitude=33.4843866,
+                    longitude=126.0,
+                ),
+                "weather_sensor",
+                fm_scheme="fm0",
+            ),
+            "weather_sensor",
+            "fm0",
+            "temperature_sensor",
+        ),
+    ],
+)
+def test_sensor_field_straightforward(
+    entity_address, entity_type, fm_scheme, exp_deserialization_name
+):
+    """Testing straightforward cases"""
+    sf = SensorField(entity_type, fm_scheme)
+    deser = sf.deserialize(entity_address, None, None)
+    assert deser.name == exp_deserialization_name
+    assert sf.serialize(entity_type, {entity_type: deser}) == entity_address
+
+
+@pytest.mark.parametrize(
+    "entity_address, entity_type, fm_scheme, error_msg",
+    [
+        (
+            "ea1.2021-01.io.flexmeasures:some.weird:identifier%that^is*not)used",
+            "market",
+            "fm0",
+            "Could not parse",
+        ),
+        (
+            "ea1.2021-01.io.flexmeasures:fm1.some.weird:identifier%that^is*not)used",
+            "market",
+            "fm1",
+            "Could not parse",
+        ),
+        (
+            build_entity_address(
+                dict(market_name="non_existing_market"), "market", fm_scheme="fm0"
+            ),
+            "market",
+            "fm0",
+            "doesn't exist",
+        ),
+        (
+            build_entity_address(dict(sensor_id=-1), "sensor", fm_scheme="fm1"),
+            "market",
+            "fm1",
+            "Could not parse",
+        ),
+        ("ea1.2021-13.io.flexmeasures:fm1.9", "sensor", "fm1", "date specification"),
+    ],
+)
+def test_sensor_field_invalid(entity_address, entity_type, fm_scheme, error_msg):
+    sf = SensorField(entity_type, fm_scheme)
+    with pytest.raises(EntityAddressValidationError) as ve:
+        sf.deserialize(entity_address, None, None)
+    assert error_msg in str(ve)

flexmeasures/api/common/schemas/tests/test_times.py

@@ -8,7 +8,7 @@
 
 
 @pytest.mark.parametrize(
-    "duration_input,exp_deserialization",
+    "duration_input, exp_deserialization",
     [
         ("PT1H", timedelta(hours=1)),
         ("PT6M", timedelta(minutes=6)),
@@ -25,7 +25,7 @@ def test_duration_field_straightforward(duration_input, exp_deserialization):
 
 
 @pytest.mark.parametrize(
-    "duration_input,exp_deserialization,grounded_timedelta",
+    "duration_input, exp_deserialization, grounded_timedelta",
     [
         ("P1M", isodate.Duration(months=1), timedelta(days=29)),
         ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)),
@@ -59,7 +59,7 @@ def test_duration_field_nominal_grounded(
 
 
 @pytest.mark.parametrize(
-    "duration_input,error_msg",
+    "duration_input, error_msg",
     [
         ("", "Unable to parse duration string"),
         ("1H", "Unable to parse duration string"),

flexmeasures/api/common/utils/api_utils.py

@@ -12,12 +12,11 @@
 
 from flexmeasures.data import db
 from flexmeasures.data.models.assets import Asset, Power
-from flexmeasures.data.models.markets import Market, Price
+from flexmeasures.data.models.markets import Price
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.weather import WeatherSensor, Weather
 from flexmeasures.data.models.user import User
 from flexmeasures.data.utils import save_to_session
-from flexmeasures.utils.entity_address_utils import parse_entity_address
 from flexmeasures.api.common.responses import (
     unrecognized_sensor,
     ResponseTuple,
@@ -296,7 +295,7 @@ def get_or_create_user_data_source(user: User) -> DataSource:
 
 def get_weather_sensor_by(
     weather_sensor_type_name: str, latitude: float = 0, longitude: float = 0
-) -> WeatherSensor:
+) -> Union[WeatherSensor, ResponseTuple]:
     """
     Search a weather sensor by type and location.
     Can create a weather sensor if needed (depends on API mode)
@@ -346,29 +345,6 @@ def get_weather_sensor_by(
     return weather_sensor
 
 
-def get_generic_asset(
-    asset_descriptor, entity_type
-) -> Union[Asset, Market, WeatherSensor, None]:
-    """
-    Get a generic asset from form information
-    # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case
-    """
-    ea = parse_entity_address(asset_descriptor, entity_type=entity_type)
-    if ea is None:
-        return None
-    if entity_type == "connection":
-        return Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none()
-    elif entity_type == "market":
-        return Market.query.filter(Market.name == ea["market_name"]).one_or_none()
-    elif entity_type == "sensor":
-        return get_weather_sensor_by(
-            ea["weather_sensor_type_name"],
-            ea["latitude"],
-            ea["longitude"],
-        )
-    return None
-
-
 def save_to_db(
     timed_values: List[Union[Power, Price, Weather]], forecasting_jobs: List[Job]
 ) -> ResponseTuple:

flexmeasures/api/common/utils/validators.py

@@ -16,6 +16,7 @@
 
 from webargs.flaskparser import parser
 
+from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.common.schemas.times import DurationField
 from flexmeasures.api.common.responses import (  # noqa: F401
     required_info_missing,
@@ -40,7 +41,6 @@
     parse_as_list,
     contains_empty_items,
     upsample_values,
-    get_generic_asset,
 )
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.config import db
@@ -728,7 +728,9 @@ def decorated_service(*args, **kwargs):
     return wrapper
 
 
-def post_data_checked_for_required_resolution(entity_type):  # noqa: C901
+def post_data_checked_for_required_resolution(
+    entity_type: str, fm_scheme: str
+):  # noqa: C901
     """Decorator which checks that a POST request receives time series data with the event resolutions
     required by the sensor (asset). It sets the "resolution" keyword argument.
     If the resolution in the data is a multiple of the asset resolution, values are upsampled to the asset resolution.
@@ -786,7 +788,9 @@ def decorated_service(*args, **kwargs):
             for asset_group in kwargs["generic_asset_name_groups"]:
                 for asset_descriptor in asset_group:
                     # Getting the asset
-                    generic_asset = get_generic_asset(asset_descriptor, entity_type)
+                    generic_asset = SensorField(entity_type, fm_scheme).deserialize(
+                        asset_descriptor
+                    )
                     if generic_asset is None:
                         return unrecognized_asset(
                             f"Failed to look up asset by {asset_descriptor}"
@@ -804,6 +808,7 @@ def decorated_service(*args, **kwargs):
                     last_asset = generic_asset
 
             # if inferred resolution is a multiple from required_solution, we can upsample_values
+            # todo: next line fails on sensors with 0 resolution
             if inferred_resolution % required_resolution == timedelta(hours=0):
                 for i in range(len(kwargs["value_groups"])):
                     kwargs["value_groups"][i] = upsample_values(
@@ -829,7 +834,7 @@ def decorated_service(*args, **kwargs):
     return wrapper
 
 
-def get_data_downsampling_allowed(entity_type):
+def get_data_downsampling_allowed(entity_type: str, fm_scheme: str):
     """Decorator which allows downsampling of data which a GET request returns.
     It checks for a form parameter "resolution".
     If that is given and is a multiple of the asset's event_resolution,
@@ -872,7 +877,9 @@ def decorated_service(*args, **kwargs):
                 # of the event_resolution(s) and thus downsampling is possible)
                 for asset_group in kwargs["generic_asset_name_groups"]:
                     for asset_descriptor in asset_group:
-                        generic_asset = get_generic_asset(asset_descriptor, entity_type)
+                        generic_asset = SensorField(entity_type, fm_scheme).deserialize(
+                            asset_descriptor
+                        )
                         if generic_asset is None:
                             return unrecognized_asset()
                         asset_resolution = generic_asset.event_resolution