From 9131fd7ca0fb3ac604e8cac4a6f37c0f35628ac2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 15:55:17 +0100 Subject: [PATCH 01/21] Rename entity type for weather sensors --- flexmeasures/api/common/utils/api_utils.py | 2 +- flexmeasures/api/v1_1/implementations.py | 4 ++-- flexmeasures/api/v2_0/implementations/sensors.py | 4 ++-- flexmeasures/api/v2_0/tests/utils.py | 2 +- flexmeasures/utils/entity_address_utils.py | 10 +++++----- flexmeasures/utils/tests/test_entity_address_utils.py | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index e56c15952..dbd40a84b 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -360,7 +360,7 @@ def get_generic_asset( return Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() elif entity_type == "market": return Market.query.filter(Market.name == ea["market_name"]).one_or_none() - elif entity_type == "sensor": + elif entity_type == "weather_sensor": return get_weather_sensor_by( ea["weather_sensor_type_name"], ea["latitude"], diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 3efdf2ffe..8b67284dd 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -140,7 +140,7 @@ def post_price_data_response( @optional_horizon_accepted(accept_repeating_interval=True) @values_required @period_required -@post_data_checked_for_required_resolution("sensor") +@post_data_checked_for_required_resolution("weather_sensor") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -162,7 +162,7 @@ def post_weather_data_response( # noqa: C901 # Parse the entity address try: - ea = parse_entity_address(sensor, entity_type="sensor") + ea = parse_entity_address(sensor, entity_type="weather_sensor") except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] diff --git a/flexmeasures/api/v2_0/implementations/sensors.py b/flexmeasures/api/v2_0/implementations/sensors.py index a4c8d6382..7d6eb4f44 100644 --- a/flexmeasures/api/v2_0/implementations/sensors.py +++ b/flexmeasures/api/v2_0/implementations/sensors.py @@ -134,7 +134,7 @@ def post_price_data_response( # noqa C901 @optional_prior_accepted() @values_required @period_required -@post_data_checked_for_required_resolution("sensor") +@post_data_checked_for_required_resolution("weather_sensor") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -160,7 +160,7 @@ def post_weather_data_response( # noqa: C901 # Parse the entity address try: - ea = parse_entity_address(sensor, entity_type="sensor") + ea = parse_entity_address(sensor, entity_type="weather_sensor") except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index b07bd0700..abc79b620 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -76,7 +76,7 @@ def verify_sensor_data_in_db( elif entity_type == "market": sensor_type = Market data_type = Price - elif entity_type == "sensor": + elif entity_type == "weather_sensor": sensor_type = WeatherSensor data_type = Weather else: diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 057dfc987..255981ed5 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -76,7 +76,7 @@ def build_field(field: str, required: bool = True): locally_unique_str = ( f"{build_field('owner_id', required=False)}{build_field('asset_id')}" ) - elif entity_type == "sensor": + elif entity_type == "weather_sensor": locally_unique_str = f"{build_field('weather_sensor_type_name')}{build_field('latitude')}{build_field('longitude')}" elif entity_type == "market": locally_unique_str = f"{build_field('market_name')}" @@ -100,8 +100,8 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: connection = ea1.2018-06.localhost:40:30 connection = ea1.2018-06.io.flexmeasures:: - sensor = ea1.2018-06.io.flexmeasures:temperature:52:73.0 - sensor = ea1.2018-06.io.flexmeasures::: + weather_sensor = ea1.2018-06.io.flexmeasures:temperature:52:73.0 + weather_sensor = ea1.2018-06.io.flexmeasures::: market = ea1.2018-06.io.flexmeasures:epex_da market = ea1.2018-06.io.flexmeasures: event = ea1.2018-06.io.flexmeasures:40:30:302:soc @@ -123,7 +123,7 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: f"After '{ADDR_SCHEME}.', a date spec of the format {date_regex} is expected." ) # Also the entity type - if entity_type not in ("connection", "sensor", "market", "event"): + if entity_type not in ("connection", "weather_sensor", "market", "event"): raise EntityAddressException(f"Unrecognized entity type: {entity_type}") if entity_type == "connection": @@ -145,7 +145,7 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: "asset_id": int, } return _typed_regex_results(match, value_types) - elif entity_type == "sensor": + elif entity_type == "weather_sensor": match = re.search( r"^" r"(?P.+)" diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index 0e83826db..b271f1abb 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -37,7 +37,7 @@ latitude=52, longitude=73.0, ), - "sensor", + "weather_sensor", "flexmeasures.io", "ea1.2021-01.io.flexmeasures:temperature:52:73.0", ), @@ -89,7 +89,7 @@ def test_build_entity_address( "date spec of the format", ), ( - "sensor", + "weather_sensor", "ea1.2018-04.localhost:5000:40:30", "Could not parse", # no sensor type (which starts with a letter) ), @@ -110,7 +110,7 @@ def test_build_entity_address( dict(naming_authority="2018-06.io.flexmeasures", owner_id=40, asset_id=30), ), ( - "sensor", + "weather_sensor", "ea1.2018-06.io.flexmeasures:temperature:-52:73.0", dict( naming_authority="2018-06.io.flexmeasures", @@ -161,7 +161,7 @@ def test_parse_entity_address(entity_type, entity_address, exp_result): assert res[field] == exp_result[field] if entity_type == "market": assert res["market_name"] == exp_result["market_name"] - if entity_type == "sensor": + if entity_type == "weather_sensor": for field in ("weather_sensor_type_name", "latitude", "longitude"): assert res[field] == exp_result[field] if entity_type == "event": From 0c497571135a3d83ea7171a888aaebdfd2d47f2a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 15:57:40 +0100 Subject: [PATCH 02/21] Update domain registration year in docstring --- flexmeasures/utils/entity_address_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 255981ed5..cb5118a89 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -98,14 +98,14 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: For example: - connection = ea1.2018-06.localhost:40:30 - connection = ea1.2018-06.io.flexmeasures:: - weather_sensor = ea1.2018-06.io.flexmeasures:temperature:52:73.0 - weather_sensor = ea1.2018-06.io.flexmeasures::: - market = ea1.2018-06.io.flexmeasures:epex_da - market = ea1.2018-06.io.flexmeasures: - event = ea1.2018-06.io.flexmeasures:40:30:302:soc - event = ea1.2018-06.io.flexmeasures:::: + connection = ea1.2021-01.localhost:40:30 + connection = ea1.2021-01.io.flexmeasures:: + weather_sensor = ea1.2021-01.io.flexmeasures:temperature:52:73.0 + weather_sensor = ea1.2021-01.io.flexmeasures::: + market = ea1.2021-01.io.flexmeasures:epex_da + market = ea1.2021-01.io.flexmeasures: + event = ea1.2021-01.io.flexmeasures:40:30:302:soc + event = ea1.2021-01.io.flexmeasures:::: Returns a dictionary with scheme, naming_authority and various other fields, depending on the entity type (see examples above). From ef54a2450fc569266ae17f897021d0d4556857c9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 16:32:32 +0100 Subject: [PATCH 03/21] Build and parse sensor entity addresses --- flexmeasures/data/models/time_series.py | 5 ++++ flexmeasures/utils/entity_address_utils.py | 29 ++++++++++++++++--- .../utils/tests/test_entity_address_utils.py | 12 ++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 08e721509..0425936f7 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -14,11 +14,16 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.utils.entity_address_utils import build_entity_address class Sensor(db.Model, tb.SensorDBMixin): """A sensor measures events. """ + @property + def entity_address(self) -> str: + return build_entity_address(dict(sensor_id=self.id), "sensor") + class TimedValue(object): """ diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index cb5118a89..231ffa101 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -72,7 +72,9 @@ def build_field(field: str, required: bool = True): return "" return f":{entity_info[field]}" - if entity_type == "connection": + if entity_type == "sensor": + locally_unique_str = f"{build_field('sensor_id')}" + elif entity_type == "connection": locally_unique_str = ( f"{build_field('owner_id', required=False)}{build_field('asset_id')}" ) @@ -87,7 +89,7 @@ def build_field(field: str, required: bool = True): return build_ea_scheme_and_naming_authority(host) + locally_unique_str -def parse_entity_address(entity_address: str, entity_type: str) -> dict: +def parse_entity_address(entity_address: str, entity_type: str) -> dict: # noqa: C901 """ Parses a generic asset name into an info dict. @@ -98,6 +100,8 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: For example: + sensor = ea1.2021-01.io.flexmeasures:42 + sensor = ea1.2021-01.io.flexmeasures: connection = ea1.2021-01.localhost:40:30 connection = ea1.2021-01.io.flexmeasures:: weather_sensor = ea1.2021-01.io.flexmeasures:temperature:52:73.0 @@ -123,10 +127,27 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: f"After '{ADDR_SCHEME}.', a date spec of the format {date_regex} is expected." ) # Also the entity type - if entity_type not in ("connection", "weather_sensor", "market", "event"): + if entity_type not in ("sensor", "connection", "weather_sensor", "market", "event"): raise EntityAddressException(f"Unrecognized entity type: {entity_type}") - if entity_type == "connection": + if entity_type == "sensor": + match = re.search( + r"^" + r"(?P.+)\." + fr"(?P{date_regex}\.[^:]+)" # everything until the colon (no port) + r":" + r"(?P\d+)" + r"$", + entity_address, + ) + if match: + value_types = { + "scheme": str, + "naming_authority": str, + "sensor_id": int, + } + return _typed_regex_results(match, value_types) + elif entity_type == "connection": match = re.search( r"^" r"(?P.+)\." diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index b271f1abb..031635523 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -13,6 +13,18 @@ @pytest.mark.parametrize( "info, entity_type, host, exp_result", [ + ( + dict(sensor_id=42), + "sensor", + "flexmeasures.io", + "ea1.2021-01.io.flexmeasures:42", + ), + ( + dict(asset_id=42), + "sensor", + "flexmeasures.io", + "required field 'sensor_id'", + ), ( dict(owner_id=3, asset_id=40), "connection", From 2d56a5ab112122abd00fc8b2a3529ec01c5548f0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:21:16 +0100 Subject: [PATCH 04/21] Fix pass-through of error response --- flexmeasures/api/common/utils/api_utils.py | 2 +- flexmeasures/api/v1_1/implementations.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index dbd40a84b..ccb9a5650 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -296,7 +296,7 @@ def get_or_create_user_data_source(user: User) -> DataSource: def get_weather_sensor_by( weather_sensor_type_name: str, latitude: float = 0, longitude: float = 0 -) -> WeatherSensor: +) -> Union[WeatherSensor, ResponseTuple]: """ Search a weather sensor by type and location. Can create a weather sensor if needed (depends on API mode) diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 8b67284dd..bc2374440 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -13,6 +13,7 @@ invalid_domain, invalid_unit, unrecognized_market, + ResponseTuple, ) from flexmeasures.api.common.utils.api_utils import ( save_to_db, @@ -177,6 +178,9 @@ def post_weather_data_response( # noqa: C901 weather_sensor = get_weather_sensor_by( weather_sensor_type_name, latitude, longitude ) + if type(weather_sensor) == ResponseTuple: + # Error message telling the user about the nearest weather sensor they can post to + return weather_sensor # Create new Weather objects for j, value in enumerate(value_group): From 5f9618365f2ddb1e86c2547cc6f05f8be667846e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:23:28 +0100 Subject: [PATCH 05/21] Add entity address properties to Market and WeatherSensor --- flexmeasures/data/models/markets.py | 5 +++++ flexmeasures/data/models/weather.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/markets.py b/flexmeasures/data/models/markets.py index 181a04efc..eb481dffe 100644 --- a/flexmeasures/data/models/markets.py +++ b/flexmeasures/data/models/markets.py @@ -6,6 +6,7 @@ from flexmeasures.data.config import db from flexmeasures.data.models.time_series import Sensor, TimedValue +from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -78,6 +79,10 @@ def __init__(self, **kwargs): if "display_name" not in kwargs: self.display_name = humanize(self.name) + @property + def entity_address(self) -> str: + return build_entity_address(dict(market_name=self.name), "market") + @property def price_unit(self) -> str: """Return the 'unit' property of the generic asset, just with a more insightful name.""" diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 83556054f..12351b9be 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -9,8 +9,9 @@ from flexmeasures.data.config import db from flexmeasures.data.models.time_series import Sensor, TimedValue -from flexmeasures.utils.geo_utils import parse_lat_lng +from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize +from flexmeasures.utils.geo_utils import parse_lat_lng class WeatherSensorType(db.Model): @@ -78,6 +79,17 @@ def __init__(self, **kwargs): self.id = new_sensor_id self.name = self.name.replace(" ", "_").lower() + @property + def entity_address(self) -> str: + return build_entity_address( + dict( + weather_sensor_type_name=self.weather_sensor_type_name, + latitude=self.latitude, + longitude=self.longitude, + ), + "weather_sensor", + ) + @property def weather_unit(self) -> float: """Return the 'unit' property of the generic asset, just with a more insightful name.""" From 0ac0634fa9cd6538c86eefd7c5f7da26b7f3056b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:46:45 +0100 Subject: [PATCH 06/21] Make test util function more flexible --- flexmeasures/api/v1_1/tests/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v1_1/tests/utils.py b/flexmeasures/api/v1_1/tests/utils.py index 5438c44f3..f468df6b8 100644 --- a/flexmeasures/api/v1_1/tests/utils.py +++ b/flexmeasures/api/v1_1/tests/utils.py @@ -1,5 +1,5 @@ """Useful test messages""" -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, Union from datetime import timedelta from isodate import duration_isoformat, parse_duration, parse_datetime @@ -51,7 +51,7 @@ def message_for_get_prognosis( def message_for_post_price_data( tile_n: int = 1, compress_n: int = 1, - duration: Optional[timedelta] = None, + duration: Optional[Union[timedelta, str]] = None, invalid_unit: bool = False, ) -> dict: """ @@ -60,7 +60,8 @@ def message_for_post_price_data( :param tile_n: Tile the price profile back to back to obtain price data for n days (default = 1). :param compress_n: Compress the price profile to obtain price data with a coarser resolution (default = 1), e.g. compress=4 leads to a resolution of 4 hours. - :param duration: Set a duration explicitly to obtain price data with a coarser or finer resolution + :param duration: timedelta or iso8601 string + Set a duration explicitly to obtain price data with a coarser or finer resolution (the default is equal to 24 hours * tile_n), e.g. (assuming tile_n=1) duration=timedelta(hours=6) leads to a resolution of 15 minutes, and duration=timedelta(hours=48) leads to a resolution of 2 hours. @@ -104,7 +105,11 @@ def message_for_post_price_data( "unit": "EUR/MWh", } if duration is not None: - message["duration"] = duration + message["duration"] = ( + duration_isoformat(duration) + if isinstance(duration, timedelta) + else duration + ) if compress_n > 1: message["values"] = message["values"][::compress_n] if invalid_unit: From f6ce2742aa9a7659a0ed7d3fd58c72fa34496a72 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:54:45 +0100 Subject: [PATCH 07/21] Add marshmallow schema for sensors --- flexmeasures/api/common/schemas/sensors.py | 88 ++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 flexmeasures/api/common/schemas/sensors.py diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py new file mode 100644 index 000000000..456128385 --- /dev/null +++ b/flexmeasures/api/common/schemas/sensors.py @@ -0,0 +1,88 @@ +from typing import Union + +from marshmallow import fields + +from flexmeasures.api import FMValidationError +from flexmeasures.api.common.utils.api_utils import get_weather_sensor_by +from flexmeasures.utils.entity_address_utils import ( + parse_entity_address, + EntityAddressException, +) +from flexmeasures.data.models.assets import Asset +from flexmeasures.data.models.markets import Market +from flexmeasures.data.models.weather import WeatherSensor +from flexmeasures.data.models.time_series import Sensor + + +class EntityAddressValidationError(FMValidationError): + status = "INVALID_DOMAIN" # USEF error status + + +class SensorField(fields.Str): + """Field that deserializes to a Sensor, Asset, Market or WeatherSensor + and serializes back to an entity address (string).""" + + def __init__( + self, + entity_type: str, + *args, + **kwargs, + ): + """ + :param entity_type: "sensor", "connection", "market" or "weather_sensor" + """ + self.entity_type = entity_type + super().__init__(*args, **kwargs) + + def _deserialize( + self, value, attr, obj, **kwargs + ) -> Union[Sensor, Asset, Market, WeatherSensor]: + """Deserialize to a Sensor, Asset, Market or WeatherSensor.""" + try: + ea = parse_entity_address(value, self.entity_type) + if self.entity_type == "sensor": + sensor = Sensor.query.filter(Sensor.id == ea["sensor_id"]).one_or_none() + if sensor is not None: + return sensor + else: + raise EntityAddressValidationError( + f"Sensor with entity address {value} doesn't exist." + ) + elif self.entity_type == "connection": + asset = Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() + if asset is not None: + return asset + else: + raise EntityAddressValidationError( + f"Asset with entity address {value} doesn't exist." + ) + elif self.entity_type == "market": + market = Market.query.filter( + Market.name == ea["market_name"] + ).one_or_none() + if market is not None: + return market + else: + raise EntityAddressValidationError( + f"Market with entity address {value} doesn't exist." + ) + elif self.entity_type == "weather_sensor": + weather_sensor = get_weather_sensor_by( + ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"] + ) + if weather_sensor is not None and isinstance( + weather_sensor, WeatherSensor + ): + return weather_sensor + else: + raise EntityAddressValidationError( + f"Weather sensor with entity address {value} doesn't exist." + ) + except EntityAddressException as eae: + raise EntityAddressValidationError(str(eae)) + + def _serialize( + self, value: Union[Sensor, Asset, Market, WeatherSensor], attr, data, **kwargs + ): + """Serialize to an entity address.""" + return value.entity_address From cd00ec679073da30634df0377fb23a949ec9a6ae Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:55:49 +0100 Subject: [PATCH 08/21] Improve test legibility --- flexmeasures/api/common/schemas/tests/test_times.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_times.py b/flexmeasures/api/common/schemas/tests/test_times.py index 475cd7a08..a6466b610 100644 --- a/flexmeasures/api/common/schemas/tests/test_times.py +++ b/flexmeasures/api/common/schemas/tests/test_times.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize( - "duration_input,exp_deserialization", + "duration_input, exp_deserialization", [ ("PT1H", timedelta(hours=1)), ("PT6M", timedelta(minutes=6)), @@ -25,7 +25,7 @@ def test_duration_field_straightforward(duration_input, exp_deserialization): @pytest.mark.parametrize( - "duration_input,exp_deserialization,grounded_timedelta", + "duration_input, exp_deserialization, grounded_timedelta", [ ("P1M", isodate.Duration(months=1), timedelta(days=29)), ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)), @@ -59,7 +59,7 @@ def test_duration_field_nominal_grounded( @pytest.mark.parametrize( - "duration_input,error_msg", + "duration_input, error_msg", [ ("", "Unable to parse duration string"), ("1H", "Unable to parse duration string"), From 76266b68fe25e0df212f9c1862cad934f5cbd64d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 17:59:07 +0100 Subject: [PATCH 09/21] Move setup of test WeatherSensors to higher conftest.py --- flexmeasures/api/v1_1/tests/conftest.py | 26 --------------------- flexmeasures/conftest.py | 30 +++++++++++++++++++++++++ flexmeasures/data/tests/conftest.py | 4 +++- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/flexmeasures/api/v1_1/tests/conftest.py b/flexmeasures/api/v1_1/tests/conftest.py index 578e357d1..ea371fa83 100644 --- a/flexmeasures/api/v1_1/tests/conftest.py +++ b/flexmeasures/api/v1_1/tests/conftest.py @@ -20,7 +20,6 @@ def setup_api_test_data(db): from flexmeasures.data.models.user import User, Role from flexmeasures.data.models.assets import Asset, AssetType - from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) @@ -98,29 +97,4 @@ def setup_api_test_data(db): power_forecasts.append(p_3) db.session.bulk_save_objects(power_forecasts) - # Create 2 weather sensors - test_sensor_type = WeatherSensorType(name="wind_speed") - db.session.add(test_sensor_type) - sensor = WeatherSensor( - name="wind_speed_sensor", - weather_sensor_type_name="wind_speed", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - unit="m/s", - ) - db.session.add(sensor) - - test_sensor_type = WeatherSensorType(name="temperature") - db.session.add(test_sensor_type) - sensor = WeatherSensor( - name="temperature_sensor", - weather_sensor_type_name="temperature", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - unit="°C", - ) - db.session.add(sensor) - print("Done setting up data for API v1.1 tests") diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index aee4f38c3..663e9f8ee 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -23,6 +23,7 @@ from flexmeasures.data.models.assets import AssetType, Asset, Power from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType """ @@ -344,6 +345,35 @@ def add_charging_station_assets(db: SQLAlchemy, setup_roles_users, setup_markets db.session.add(bidirectional_charging_station) +@pytest.fixture(scope="function", autouse=True) +def add_weather_sensors(db: SQLAlchemy): + """Add some weather sensors and weather sensor types.""" + + test_sensor_type = WeatherSensorType(name="wind_speed") + db.session.add(test_sensor_type) + sensor = WeatherSensor( + name="wind_speed_sensor", + weather_sensor_type_name="wind_speed", + event_resolution=timedelta(minutes=5), + latitude=33.4843866, + longitude=126, + unit="m/s", + ) + db.session.add(sensor) + + test_sensor_type = WeatherSensorType(name="temperature") + db.session.add(test_sensor_type) + sensor = WeatherSensor( + name="temperature_sensor", + weather_sensor_type_name="temperature", + event_resolution=timedelta(minutes=5), + latitude=33.4843866, + longitude=126.0, + unit="°C", + ) + db.session.add(sensor) + + @pytest.fixture(scope="function", autouse=True) def clean_redis(app): failed = app.queues["forecasting"].failed_job_registry diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 2a9fb4c48..b2ce954c2 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -45,7 +45,9 @@ def add_test_weather_sensor_and_forecasts(db: SQLAlchemy): name="Seita", type="demo script" ).one_or_none() for sensor_name in ("radiation", "wind_speed"): - sensor_type = WeatherSensorType(name=sensor_name) + sensor_type = WeatherSensorType.query.filter_by(name=sensor_name).one_or_none() + if sensor_type is None: + sensor_type = WeatherSensorType(name=sensor_name) sensor = WeatherSensor( name=sensor_name, sensor_type=sensor_type, latitude=100, longitude=100 ) From 21b458bead108f67abfa5a5e167886bc6945a112 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 20:15:21 +0100 Subject: [PATCH 10/21] Better regex for date specification --- flexmeasures/utils/entity_address_utils.py | 4 ++-- flexmeasures/utils/tests/test_entity_address_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 231ffa101..07e84a616 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -121,10 +121,10 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: # noqa raise EntityAddressException( f"A valid type 1 USEF entity address starts with '{ADDR_SCHEME}', please review {entity_address}" ) - date_regex = r"[0-9]{4}-[0-9]{2}" + date_regex = r"([0-9]{4})-(0[1-9]|1[012])" if not re.search(fr"^{date_regex}$", entity_address[4:11]): raise EntityAddressException( - f"After '{ADDR_SCHEME}.', a date spec of the format {date_regex} is expected." + f"After '{ADDR_SCHEME}.', a date specification of the format {date_regex} is expected." ) # Also the entity type if entity_type not in ("sensor", "connection", "weather_sensor", "market", "event"): diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index 031635523..85d0357fb 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -98,7 +98,7 @@ def test_build_entity_address( ( "connection", "ea1.2018-RR.localhost:40:30", - "date spec of the format", + "date specification", ), ( "weather_sensor", From 68f4bd3fd1acf632547967f7d37127248b5e43bd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 20:18:59 +0100 Subject: [PATCH 11/21] Test marshmallow schema for sensors --- .../api/common/schemas/tests/test_sensors.py | 72 +++++++++++++++++++ flexmeasures/conftest.py | 11 +++ flexmeasures/utils/entity_address_utils.py | 18 +++-- 3 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 flexmeasures/api/common/schemas/tests/test_sensors.py diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py new file mode 100644 index 000000000..9dd73e0fa --- /dev/null +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -0,0 +1,72 @@ +import pytest + +from flexmeasures.api.common.schemas.sensors import ( + SensorField, + EntityAddressValidationError, +) +from flexmeasures.utils.entity_address_utils import build_entity_address + + +@pytest.mark.parametrize( + "entity_address, entity_type, exp_deserialization_name", + [ + ( + build_entity_address(dict(sensor_id=9), "sensor"), + "sensor", + "my daughter's height", + ), + ( + build_entity_address(dict(market_name="epex_da"), "market"), + "market", + "epex_da", + ), + ( + build_entity_address(dict(owner_id=1, asset_id=3), "connection"), + "connection", + "Test battery with no known prices", + ), + ( + build_entity_address( + dict( + weather_sensor_type_name="temperature", + latitude=33.4843866, + longitude=126.0, + ), + "weather_sensor", + ), + "weather_sensor", + "temperature_sensor", + ), + ], +) +def test_sensor_field_straightforward( + entity_address, entity_type, exp_deserialization_name +): + """Testing straightforward cases""" + sf = SensorField(entity_type) + deser = sf.deserialize(entity_address, None, None) + assert deser.name == exp_deserialization_name + assert sf.serialize(entity_type, {entity_type: deser}) == entity_address + + +@pytest.mark.parametrize( + "entity_address, entity_type, error_msg", + [ + ( + "ea1.2021-01.io.flexmeasures:some.weird:identifier%that^is*not)used", + "market", + "Could not parse", + ), + ( + build_entity_address(dict(market_name="non_existing_market"), "market"), + "market", + "doesn't exist", + ), + ("ea1.2021-13.io.flexmeasures:9", "sensor", "date specification"), + ], +) +def test_sensor_field_invalid(entity_address, entity_type, error_msg): + sf = SensorField(entity_type) + with pytest.raises(EntityAddressValidationError) as ve: + sf.deserialize(entity_address, None, None) + assert error_msg in str(ve) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 663e9f8ee..20a1eceab 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -24,6 +24,7 @@ from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.markets import Market, Price from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType +from flexmeasures.data.models.time_series import Sensor """ @@ -374,6 +375,16 @@ def add_weather_sensors(db: SQLAlchemy): db.session.add(sensor) +@pytest.fixture(scope="function", autouse=True) +def add_sensors(db: SQLAlchemy): + """Add some generic sensors.""" + height_sensor = Sensor( + name="my daughter's height", + unit="m", + ) + db.session.add(height_sensor) + + @pytest.fixture(scope="function", autouse=True) def clean_redis(app): failed = app.queues["forecasting"].failed_job_registry diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 07e84a616..c92355a6f 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -59,9 +59,10 @@ def build_entity_address( Returns the address as string. """ if host is None: - # TODO: Assume localhost if request is not given either (for tests/simulations), - # or should we raise? - host = urlparse(request.url).netloc + try: + host = urlparse(request.url).netloc + except RuntimeError: + host = "localhost" def build_field(field: str, required: bool = True): if required and field not in entity_info: @@ -89,7 +90,10 @@ def build_field(field: str, required: bool = True): return build_ea_scheme_and_naming_authority(host) + locally_unique_str -def parse_entity_address(entity_address: str, entity_type: str) -> dict: # noqa: C901 +def parse_entity_address( # noqa: C901 + entity_address: str, + entity_type: str, +) -> dict: """ Parses a generic asset name into an info dict. @@ -254,14 +258,14 @@ def build_ea_scheme_and_naming_authority( [domain_parts.subdomain, domain_parts.domain, domain_parts.suffix], ) ) - if config_var_domain_key in current_app.config.get( + if domain_parts.domain in ("localhost", "127.0.0.1"): + host_auth_start_month = get_first_day_of_next_month().strftime("%Y-%m") + elif config_var_domain_key in current_app.config.get( "FLEXMEASURES_HOSTS_AND_AUTH_START", {} ): host_auth_start_month = current_app.config.get( "FLEXMEASURES_HOSTS_AND_AUTH_START", {} )[config_var_domain_key] - elif domain_parts.domain in ("localhost", "127.0.0.1"): - host_auth_start_month = get_first_day_of_next_month().strftime("%Y-%m") else: raise Exception( f"Could not find out when authority for {config_var_domain_key} started. Is FLEXMEASURES_HOSTS_AND_AUTH_START configured for it?" From 23ea22f5f03b9d8317e450b3054b6bcb490ae9ec Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Mar 2021 20:52:04 +0100 Subject: [PATCH 12/21] mypy --- flexmeasures/api/common/schemas/sensors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index 456128385..59c7c9367 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -80,6 +80,7 @@ def _deserialize( ) except EntityAddressException as eae: raise EntityAddressValidationError(str(eae)) + return NotImplemented def _serialize( self, value: Union[Sensor, Asset, Market, WeatherSensor], attr, data, **kwargs From 577bc1ca32c7292170792373f2e48ad8efa653e3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 Mar 2021 11:22:46 +0100 Subject: [PATCH 13/21] Fix variable naming of test util --- flexmeasures/api/v2_0/tests/utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index abc79b620..f1a1054de 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -1,5 +1,4 @@ -from flexmeasures.data.services.users import find_user_by_email -from typing import Optional +from typing import Optional, Union from datetime import timedelta from isodate import duration_isoformat, parse_duration, parse_datetime @@ -10,6 +9,7 @@ from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.models.markets import Market, Price from flexmeasures.data.models.weather import WeatherSensor, Weather +from flexmeasures.data.services.users import find_user_by_email from flexmeasures.api.v1_1.tests.utils import ( message_for_post_price_data as v1_1_message_for_post_price_data, ) @@ -84,8 +84,10 @@ def verify_sensor_data_in_db( start = parse_datetime(post_message["start"]) end = start + parse_duration(post_message["duration"]) - market: Market = get_generic_asset(post_message[entity_type], entity_type) - resolution = market.event_resolution + sensor: Union[Asset, Market, WeatherSensor] = get_generic_asset( + post_message[entity_type], entity_type + ) + resolution = sensor.event_resolution if "horizon" in post_message: horizon = parse_duration(post_message["horizon"]) query = ( @@ -95,7 +97,7 @@ def verify_sensor_data_in_db( ) .filter(data_type.horizon == horizon) .join(sensor_type) - .filter(sensor_type.name == market.name) + .filter(sensor_type.name == sensor.name) ) else: query = ( @@ -109,7 +111,7 @@ def verify_sensor_data_in_db( ) # .filter(data_type.horizon == (data_type.datetime + resolution) - prior) # only for sensors with 0-hour ex_post knowledge horizon function .join(sensor_type) - .filter(sensor_type.name == market.name) + .filter(sensor_type.name == sensor.name) ) # todo: after basing Price on TimedBelief, we should be able to get a BeliefsDataFrame from the query directly df = pd.DataFrame( @@ -122,7 +124,7 @@ def verify_sensor_data_in_db( "horizon": "belief_horizon", } ) - bdf = tb.BeliefsDataFrame(df, sensor=market, source="Some source") + bdf = tb.BeliefsDataFrame(df, sensor=sensor, source="Some source") if "prior" in post_message: prior = parse_datetime(post_message["prior"]) bdf = bdf.fixed_viewpoint(prior) From 7a7e26df991946f201bb47a2e2e86332c8c00e5c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Apr 2021 16:24:36 +0200 Subject: [PATCH 14/21] Update CLI command --- flexmeasures/data/scripts/cli_tasks/data_add.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index d5152a1c3..7107c289e 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -158,9 +158,8 @@ def add_weather_sensor(**args): sensor = WeatherSensor(**args) app.db.session.add(sensor) app.db.session.commit() - print(f"Successfully created sensor with ID:{sensor.id}.") - # TODO: uncomment when #66 has landed - # print(f" You can access it at its entity address {sensor.entity_address}") + print(f"Successfully created weather sensor with ID:{sensor.id}.") + print(f" You can access it at its entity address {sensor.entity_address}") @fm_add_data.command("structure") From b5f7f450ffbe5a40b92e79be29ff23136f69d874 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 2 Apr 2021 15:11:24 +0200 Subject: [PATCH 15/21] Fix tests with deprecation of sqlalchemy RowProxy in 1.4 --- flexmeasures/data/queries/utils.py | 27 ------------------------- flexmeasures/data/scripts/data_gen.py | 5 +++-- flexmeasures/data/services/resources.py | 15 +++++++------- 3 files changed, 10 insertions(+), 37 deletions(-) diff --git a/flexmeasures/data/queries/utils.py b/flexmeasures/data/queries/utils.py index 0fb54a421..e2c025cba 100644 --- a/flexmeasures/data/queries/utils.py +++ b/flexmeasures/data/queries/utils.py @@ -5,7 +5,6 @@ import timely_beliefs as tb from sqlalchemy.orm import Query, Session -from sqlalchemy.engine.result import RowProxy from flexmeasures.data.config import db from flexmeasures.data.models.data_sources import DataSource @@ -158,32 +157,6 @@ def add_belief_timing_filter( return query -def parse_sqlalchemy_results(results: List[RowProxy]) -> List[dict]: - """ - Returns a list of dicts, whose keys are column names. E.g.: - - data = session.execute("Select latitude from asset;").fetchall() - for row in parse_sqlalchemy_results(data): - print("------------") - for key, val in row: - print f"{key}: {val}" - - """ - parsed_results: List[dict] = [] - - if len(results) == 0: - return parsed_results - - # results from SQLAlchemy are returned as a list of tuples; - # this procedure converts it into a list of dicts - for row_number, row in enumerate(results): - parsed_results.append({}) - for column_number, value in enumerate(row): - parsed_results[row_number][row.keys()[column_number]] = value - - return parsed_results - - def simplify_index( bdf: tb.BeliefsDataFrame, index_levels_to_columns: Optional[List[str]] = None ) -> pd.DataFrame: diff --git a/flexmeasures/data/scripts/data_gen.py b/flexmeasures/data/scripts/data_gen.py index a71a7e0b3..700838bd2 100644 --- a/flexmeasures/data/scripts/data_gen.py +++ b/flexmeasures/data/scripts/data_gen.py @@ -19,13 +19,13 @@ import inflect from flexmeasures.data.models.markets import MarketType, Market, Price +from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.models.assets import AssetType, Asset, Power from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.weather import WeatherSensorType, WeatherSensor, Weather from flexmeasures.data.models.user import User, Role, RolesUsers from flexmeasures.data.models.forecasting import lookup_model_specs_configurator from flexmeasures.data.models.forecasting.exceptions import NotEnoughDataException -from flexmeasures.data.queries.utils import parse_sqlalchemy_results from flexmeasures.utils.time_utils import ensure_local_timezone from flexmeasures.data.transactional import as_transaction @@ -641,7 +641,7 @@ def load_tables( affected_classes = get_affected_classes(structure, data) statement = "SELECT sequence_name from information_schema.sequences;" data = db.session.execute(statement).fetchall() - sequence_names = [s["sequence_name"] for s in parse_sqlalchemy_results(data)] + sequence_names = [s.sequence_name for s in data] for c in affected_classes: file_path = "%s/%s/%s.obj" % (backup_path, backup_name, c.__tablename__) sequence_name = "%s_id_seq" % c.__tablename__ @@ -682,6 +682,7 @@ def get_affected_classes(structure: bool = True, data: bool = False) -> List: Role, User, RolesUsers, + Sensor, MarketType, Market, AssetType, diff --git a/flexmeasures/data/services/resources.py b/flexmeasures/data/services/resources.py index 6fc21eee7..f66f7fabf 100644 --- a/flexmeasures/data/services/resources.py +++ b/flexmeasures/data/services/resources.py @@ -13,6 +13,7 @@ import inflect import pandas as pd from sqlalchemy.orm import Query, Session +from sqlalchemy.engine import Row import timely_beliefs as tb from flexmeasures.data.models.assets import ( @@ -24,7 +25,7 @@ from flexmeasures.data.models.markets import Market, Price from flexmeasures.data.models.weather import Weather, WeatherSensor, WeatherSensorType from flexmeasures.data.models.user import User -from flexmeasures.data.queries.utils import simplify_index, parse_sqlalchemy_results +from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.services.time_series import aggregate_values from flexmeasures.utils.geo_utils import parse_lat_lng from flexmeasures.utils import coding_utils, time_utils @@ -207,16 +208,14 @@ def get_center_location(db: Session, user: Optional[User]) -> Tuple[float, float ) if user and not user.has_role("admin"): query += f" where owner_id = {user.id}" - locations: List[dict] = parse_sqlalchemy_results( - db.session.execute(query + ";").fetchall() - ) + locations: List[Row] = db.session.execute(query + ";").fetchall() if ( len(locations) == 0 - or locations[0]["latitude"] is None - or locations[0]["longitude"] is None + or locations[0].latitude is None + or locations[0].longitude is None ): - return (52.38, 4.88) # Amsterdam, NL - return locations[0]["latitude"], locations[0]["longitude"] + return 52.366, 4.904 # Amsterdam, NL + return locations[0].latitude, locations[0].longitude def check_cache(attribute): From d651b19f7250d5d68426129756cc25288e96de51 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 3 Apr 2021 17:42:58 +0200 Subject: [PATCH 16/21] Prefer localhost over 127.0.0.1 in entity addresses and strip www. --- flexmeasures/utils/entity_address_utils.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index af96b1981..7bbd56c0d 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -48,6 +48,20 @@ class EntityAddressException(Exception): pass +def get_host() -> str: + """Get host from the context of the request. + + Strips off www. but keeps subdomains. + Can be localhost, too. + """ + if has_request_context(): + host = urlparse(request.url).netloc.lstrip("www.") + if host[:9] != "127.0.0.1": + return host + # Assume localhost (for CLI/tests/simulations) + return "localhost" + + def build_entity_address( entity_info: dict, entity_type: str, host: Optional[str] = None ) -> str: @@ -60,11 +74,7 @@ def build_entity_address( Returns the address as string. """ if host is None: - if has_request_context(): - host = urlparse(request.url).netloc - else: - # Assume localhost (for CLI/tests/simulations) - host = "localhost" + host = get_host() def build_field(field: str, required: bool = True): if required and field not in entity_info: From 648e248f786066a12dad81dfc22308eef0190472 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 3 Apr 2021 19:49:27 +0200 Subject: [PATCH 17/21] Introduce fm1 scheme for local part of entity addresses --- flexmeasures/api/common/schemas/sensors.py | 126 +++++++++++------ .../api/common/schemas/tests/test_sensors.py | 45 ++++-- flexmeasures/api/common/utils/api_utils.py | 26 +--- flexmeasures/api/common/utils/validators.py | 17 ++- flexmeasures/api/v1/implementations.py | 10 +- flexmeasures/api/v1_1/implementations.py | 14 +- flexmeasures/api/v1_1/tests/test_api_v1_1.py | 8 +- flexmeasures/api/v1_1/tests/utils.py | 4 +- flexmeasures/api/v1_2/implementations.py | 6 +- flexmeasures/api/v1_3/implementations.py | 6 +- .../api/v2_0/implementations/sensors.py | 20 +-- .../api/v2_0/tests/test_api_v2_0_sensors.py | 13 +- flexmeasures/api/v2_0/tests/utils.py | 26 ++-- flexmeasures/data/models/assets.py | 12 +- flexmeasures/data/models/markets.py | 10 +- flexmeasures/data/models/weather.py | 12 +- flexmeasures/ui/templates/crud/assets.html | 4 + flexmeasures/utils/entity_address_utils.py | 133 ++++++++++++++---- .../utils/tests/test_entity_address_utils.py | 52 +++++-- 19 files changed, 377 insertions(+), 167 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index 59c7c9367..f39d61cff 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -22,62 +22,105 @@ class SensorField(fields.Str): """Field that deserializes to a Sensor, Asset, Market or WeatherSensor and serializes back to an entity address (string).""" + # todo: when Actuators also get an entity address, refactor this class to EntityField, + # where an Entity represents anything with an entity address: we currently foresee Sensors and Actuators + def __init__( self, entity_type: str, + fm_scheme: str, *args, **kwargs, ): """ :param entity_type: "sensor", "connection", "market" or "weather_sensor" + :param fm_scheme: "fm0" or "fm1" """ self.entity_type = entity_type + self.fm_scheme = fm_scheme super().__init__(*args, **kwargs) - def _deserialize( + def _deserialize( # noqa: C901 todo: the noqa can probably be removed after refactoring Asset/Market/WeatherSensor to Sensor self, value, attr, obj, **kwargs ) -> Union[Sensor, Asset, Market, WeatherSensor]: """Deserialize to a Sensor, Asset, Market or WeatherSensor.""" + # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case try: - ea = parse_entity_address(value, self.entity_type) - if self.entity_type == "sensor": - sensor = Sensor.query.filter(Sensor.id == ea["sensor_id"]).one_or_none() - if sensor is not None: - return sensor - else: - raise EntityAddressValidationError( - f"Sensor with entity address {value} doesn't exist." - ) - elif self.entity_type == "connection": - asset = Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() - if asset is not None: - return asset - else: - raise EntityAddressValidationError( - f"Asset with entity address {value} doesn't exist." - ) - elif self.entity_type == "market": - market = Market.query.filter( - Market.name == ea["market_name"] - ).one_or_none() - if market is not None: - return market - else: - raise EntityAddressValidationError( - f"Market with entity address {value} doesn't exist." - ) - elif self.entity_type == "weather_sensor": - weather_sensor = get_weather_sensor_by( - ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"] - ) - if weather_sensor is not None and isinstance( - weather_sensor, WeatherSensor - ): - return weather_sensor - else: - raise EntityAddressValidationError( - f"Weather sensor with entity address {value} doesn't exist." + ea = parse_entity_address(value, self.entity_type, self.fm_scheme) + if self.fm_scheme == "fm0": + if self.entity_type == "connection": + asset = Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() + if asset is not None: + return asset + else: + raise EntityAddressValidationError( + f"Asset with entity address {value} doesn't exist." + ) + elif self.entity_type == "market": + market = Market.query.filter( + Market.name == ea["market_name"] + ).one_or_none() + if market is not None: + return market + else: + raise EntityAddressValidationError( + f"Market with entity address {value} doesn't exist." + ) + elif self.entity_type == "weather_sensor": + weather_sensor = get_weather_sensor_by( + ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"] ) + if weather_sensor is not None and isinstance( + weather_sensor, WeatherSensor + ): + return weather_sensor + else: + raise EntityAddressValidationError( + f"Weather sensor with entity address {value} doesn't exist." + ) + else: + if self.entity_type == "sensor": + sensor = Sensor.query.filter( + Sensor.id == ea["sensor_id"] + ).one_or_none() + if sensor is not None: + return sensor + else: + raise EntityAddressValidationError( + f"Sensor with entity address {value} doesn't exist." + ) + elif self.entity_type == "connection": + asset = Asset.query.filter( + Asset.id == ea["sensor_id"] + ).one_or_none() + if asset is not None: + return asset + else: + raise EntityAddressValidationError( + f"Asset with entity address {value} doesn't exist." + ) + elif self.entity_type == "market": + market = Market.query.filter( + Market.id == ea["sensor_id"] + ).one_or_none() + if market is not None: + return market + else: + raise EntityAddressValidationError( + f"Market with entity address {value} doesn't exist." + ) + elif self.entity_type == "weather_sensor": + weather_sensor = WeatherSensor.query.filter( + WeatherSensor.id == ea["sensor_id"] + ).one_or_none() + if weather_sensor is not None and isinstance( + weather_sensor, WeatherSensor + ): + return weather_sensor + else: + raise EntityAddressValidationError( + f"Weather sensor with entity address {value} doesn't exist." + ) except EntityAddressException as eae: raise EntityAddressValidationError(str(eae)) return NotImplemented @@ -86,4 +129,7 @@ def _serialize( self, value: Union[Sensor, Asset, Market, WeatherSensor], attr, data, **kwargs ): """Serialize to an entity address.""" - return value.entity_address + if self.fm_scheme == "fm0": + return value.entity_address_fm0 + else: + return value.entity_address diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py index 9dd73e0fa..949952bcc 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensors.py +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -8,21 +8,28 @@ @pytest.mark.parametrize( - "entity_address, entity_type, exp_deserialization_name", + "entity_address, entity_type, fm_scheme, exp_deserialization_name", [ ( build_entity_address(dict(sensor_id=9), "sensor"), "sensor", + "fm1", "my daughter's height", ), ( - build_entity_address(dict(market_name="epex_da"), "market"), + build_entity_address( + dict(market_name="epex_da"), "market", fm_scheme="fm0" + ), "market", + "fm0", "epex_da", ), ( - build_entity_address(dict(owner_id=1, asset_id=3), "connection"), + build_entity_address( + dict(owner_id=1, asset_id=3), "connection", fm_scheme="fm0" + ), "connection", + "fm0", "Test battery with no known prices", ), ( @@ -33,40 +40,58 @@ longitude=126.0, ), "weather_sensor", + fm_scheme="fm0", ), "weather_sensor", + "fm0", "temperature_sensor", ), ], ) def test_sensor_field_straightforward( - entity_address, entity_type, exp_deserialization_name + entity_address, entity_type, fm_scheme, exp_deserialization_name ): """Testing straightforward cases""" - sf = SensorField(entity_type) + sf = SensorField(entity_type, fm_scheme) deser = sf.deserialize(entity_address, None, None) assert deser.name == exp_deserialization_name assert sf.serialize(entity_type, {entity_type: deser}) == entity_address @pytest.mark.parametrize( - "entity_address, entity_type, error_msg", + "entity_address, entity_type, fm_scheme, error_msg", [ ( "ea1.2021-01.io.flexmeasures:some.weird:identifier%that^is*not)used", "market", + "fm0", + "Could not parse", + ), + ( + "ea1.2021-01.io.flexmeasures:fm1.some.weird:identifier%that^is*not)used", + "market", + "fm1", "Could not parse", ), ( - build_entity_address(dict(market_name="non_existing_market"), "market"), + build_entity_address( + dict(market_name="non_existing_market"), "market", fm_scheme="fm0" + ), "market", + "fm0", "doesn't exist", ), - ("ea1.2021-13.io.flexmeasures:9", "sensor", "date specification"), + ( + build_entity_address(dict(sensor_id=-1), "sensor", fm_scheme="fm1"), + "market", + "fm1", + "Could not parse", + ), + ("ea1.2021-13.io.flexmeasures:fm1.9", "sensor", "fm1", "date specification"), ], ) -def test_sensor_field_invalid(entity_address, entity_type, error_msg): - sf = SensorField(entity_type) +def test_sensor_field_invalid(entity_address, entity_type, fm_scheme, error_msg): + sf = SensorField(entity_type, fm_scheme) with pytest.raises(EntityAddressValidationError) as ve: sf.deserialize(entity_address, None, None) assert error_msg in str(ve) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index ccb9a5650..91d2a2f6e 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -12,12 +12,11 @@ from flexmeasures.data import db from flexmeasures.data.models.assets import Asset, Power -from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.markets import Price from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.weather import WeatherSensor, Weather from flexmeasures.data.models.user import User from flexmeasures.data.utils import save_to_session -from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( unrecognized_sensor, ResponseTuple, @@ -346,29 +345,6 @@ def get_weather_sensor_by( return weather_sensor -def get_generic_asset( - asset_descriptor, entity_type -) -> Union[Asset, Market, WeatherSensor, None]: - """ - Get a generic asset from form information - # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case - """ - ea = parse_entity_address(asset_descriptor, entity_type=entity_type) - if ea is None: - return None - if entity_type == "connection": - return Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() - elif entity_type == "market": - return Market.query.filter(Market.name == ea["market_name"]).one_or_none() - elif entity_type == "weather_sensor": - return get_weather_sensor_by( - ea["weather_sensor_type_name"], - ea["latitude"], - ea["longitude"], - ) - return None - - def save_to_db( timed_values: List[Union[Power, Price, Weather]], forecasting_jobs: List[Job] ) -> ResponseTuple: diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index 6fe7b318c..74da4c030 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -16,6 +16,7 @@ from webargs.flaskparser import parser +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.common.schemas.times import DurationField from flexmeasures.api.common.responses import ( # noqa: F401 required_info_missing, @@ -40,7 +41,6 @@ parse_as_list, contains_empty_items, upsample_values, - get_generic_asset, ) from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.config import db @@ -728,7 +728,9 @@ def decorated_service(*args, **kwargs): return wrapper -def post_data_checked_for_required_resolution(entity_type): # noqa: C901 +def post_data_checked_for_required_resolution( + entity_type: str, fm_scheme: str +): # noqa: C901 """Decorator which checks that a POST request receives time series data with the event resolutions required by the sensor (asset). It sets the "resolution" keyword argument. If the resolution in the data is a multiple of the asset resolution, values are upsampled to the asset resolution. @@ -786,7 +788,9 @@ def decorated_service(*args, **kwargs): for asset_group in kwargs["generic_asset_name_groups"]: for asset_descriptor in asset_group: # Getting the asset - generic_asset = get_generic_asset(asset_descriptor, entity_type) + generic_asset = SensorField(entity_type, fm_scheme).deserialize( + asset_descriptor + ) if generic_asset is None: return unrecognized_asset( f"Failed to look up asset by {asset_descriptor}" @@ -804,6 +808,7 @@ def decorated_service(*args, **kwargs): last_asset = generic_asset # if inferred resolution is a multiple from required_solution, we can upsample_values + # todo: next line fails on sensors with 0 resolution if inferred_resolution % required_resolution == timedelta(hours=0): for i in range(len(kwargs["value_groups"])): kwargs["value_groups"][i] = upsample_values( @@ -829,7 +834,7 @@ def decorated_service(*args, **kwargs): return wrapper -def get_data_downsampling_allowed(entity_type): +def get_data_downsampling_allowed(entity_type: str, fm_scheme: str): """Decorator which allows downsampling of data which a GET request returns. It checks for a form parameter "resolution". If that is given and is a multiple of the asset's event_resolution, @@ -872,7 +877,9 @@ def decorated_service(*args, **kwargs): # of the event_resolution(s) and thus downsampling is possible) for asset_group in kwargs["generic_asset_name_groups"]: for asset_descriptor in asset_group: - generic_asset = get_generic_asset(asset_descriptor, entity_type) + generic_asset = SensorField(entity_type, fm_scheme).deserialize( + asset_descriptor + ) if generic_asset is None: return unrecognized_asset() asset_resolution = generic_asset.event_resolution diff --git a/flexmeasures/api/v1/implementations.py b/flexmeasures/api/v1/implementations.py index 373677ddb..0940ed819 100644 --- a/flexmeasures/api/v1/implementations.py +++ b/flexmeasures/api/v1/implementations.py @@ -50,7 +50,7 @@ ) @optional_prior_accepted(ex_post=True, infer_missing=False) @period_required -@get_data_downsampling_allowed("connection") +@get_data_downsampling_allowed("connection", "fm0") @as_json def get_meter_data_response( unit, @@ -99,7 +99,7 @@ def get_meter_data_response( @values_required @optional_horizon_accepted(ex_post=True, accept_repeating_interval=True) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm0") @as_json def post_meter_data_response( unit, @@ -182,7 +182,7 @@ def collect_connection_and_value_groups( # Parse the entity address try: connection_details = parse_entity_address( - connection, entity_type="connection" + connection, entity_type="connection", fm_scheme="fm0" ) except EntityAddressException as eae: return invalid_domain(str(eae)) @@ -255,7 +255,9 @@ def create_connection_and_value_groups( # noqa: C901 # TODO: get asset through util function after refactoring # Parse the entity address try: - connection = parse_entity_address(connection, entity_type="connection") + connection = parse_entity_address( + connection, entity_type="connection", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = connection["asset_id"] diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index bc2374440..349adc0b9 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -66,7 +66,7 @@ def get_connection_response(): @optional_horizon_accepted(accept_repeating_interval=True) @values_required @period_required -@post_data_checked_for_required_resolution("market") +@post_data_checked_for_required_resolution("market", "fm0") def post_price_data_response( unit, generic_asset_name_groups, @@ -88,7 +88,7 @@ def post_price_data_response( # Parse the entity address try: - ea = parse_entity_address(market, entity_type="market") + ea = parse_entity_address(market, entity_type="market", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) market_name = ea["market_name"] @@ -141,7 +141,7 @@ def post_price_data_response( @optional_horizon_accepted(accept_repeating_interval=True) @values_required @period_required -@post_data_checked_for_required_resolution("weather_sensor") +@post_data_checked_for_required_resolution("weather_sensor", "fm0") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -163,7 +163,9 @@ def post_weather_data_response( # noqa: C901 # Parse the entity address try: - ea = parse_entity_address(sensor, entity_type="weather_sensor") + ea = parse_entity_address( + sensor, entity_type="weather_sensor", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] @@ -228,7 +230,7 @@ def post_weather_data_response( # noqa: C901 @optional_horizon_accepted(infer_missing=False, accept_repeating_interval=True) @optional_prior_accepted(infer_missing=False) @period_required -@get_data_downsampling_allowed("connection") +@get_data_downsampling_allowed("connection", "fm0") @as_json def get_prognosis_response( unit, @@ -269,7 +271,7 @@ def get_prognosis_response( @values_required @optional_horizon_accepted(ex_post=False, accept_repeating_interval=True) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm0") @as_json def post_prognosis_response( unit, diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1.py b/flexmeasures/api/v1_1/tests/test_api_v1_1.py index 0ce5519c7..f91c26915 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1.py @@ -4,6 +4,7 @@ from isodate import duration_isoformat from iso8601 import parse_date +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( request_processed, @@ -13,7 +14,6 @@ ) from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.common.utils.api_utils import ( - get_generic_asset, message_replace_name_with_ea, ) from flexmeasures.api.v1_1.tests.utils import ( @@ -152,7 +152,7 @@ def test_post_price_data(db, app, post_message): ) # only one market is affected, but two horizons horizons = [timedelta(hours=24), timedelta(hours=48)] jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", "fm0").deserialize(post_message["market"]) for job, horizon in zip(jobs, horizons): assert job.kwargs["horizon"] == horizon assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon @@ -178,8 +178,8 @@ def test_post_price_data_invalid_unit(client, post_message): print("Server responded with:\n%s" % post_price_data_response.json) assert post_price_data_response.status_code == 400 assert post_price_data_response.json["type"] == "PostPriceDataResponse" - market = parse_entity_address(post_message["market"], "market") - market_name = market["market_name"] + ea = parse_entity_address(post_message["market"], "market", fm_scheme="fm0") + market_name = ea["market_name"] market = Market.query.filter_by(name=market_name).one_or_none() assert ( post_price_data_response.json["message"] diff --git a/flexmeasures/api/v1_1/tests/utils.py b/flexmeasures/api/v1_1/tests/utils.py index f468df6b8..9c17ba0cb 100644 --- a/flexmeasures/api/v1_1/tests/utils.py +++ b/flexmeasures/api/v1_1/tests/utils.py @@ -6,7 +6,7 @@ import pandas as pd from numpy import tile -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.models.markets import Market, Price @@ -149,7 +149,7 @@ def verify_prices_in_db(post_message, values, db, swapped_sign: bool = False): start = parse_datetime(post_message["start"]) end = start + parse_duration(post_message["duration"]) horizon = parse_duration(post_message["horizon"]) - market: Market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", "fm0").deserialize(post_message["market"]) resolution = market.event_resolution query = ( db.session.query(Price.value, Price.horizon) diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py index 6fe1a0c79..2d0697eaa 100644 --- a/flexmeasures/api/v1_2/implementations.py +++ b/flexmeasures/api/v1_2/implementations.py @@ -64,7 +64,7 @@ def get_device_message_response(generic_asset_name_groups, duration): # Parse the entity address try: - ea = parse_entity_address(event, entity_type="event") + ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = ea["asset_id"] @@ -155,7 +155,9 @@ def post_udi_event_response(unit): # noqa: C901 if "event" not in form: return invalid_domain("No event identifier sent.") try: - ea = parse_entity_address(form.get("event"), entity_type="event") + ea = parse_entity_address( + form.get("event"), entity_type="event", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py index a691a5546..42302f9e2 100644 --- a/flexmeasures/api/v1_3/implementations.py +++ b/flexmeasures/api/v1_3/implementations.py @@ -69,7 +69,7 @@ def get_device_message_response(generic_asset_name_groups, duration): # Parse the entity address try: - ea = parse_entity_address(event, entity_type="event") + ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = ea["asset_id"] @@ -212,7 +212,9 @@ def post_udi_event_response(unit): if "event" not in form: return invalid_domain("No event identifier sent.") try: - ea = parse_entity_address(form.get("event"), entity_type="event") + ea = parse_entity_address( + form.get("event"), entity_type="event", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) diff --git a/flexmeasures/api/v2_0/implementations/sensors.py b/flexmeasures/api/v2_0/implementations/sensors.py index 7d6eb4f44..715b300b3 100644 --- a/flexmeasures/api/v2_0/implementations/sensors.py +++ b/flexmeasures/api/v2_0/implementations/sensors.py @@ -50,7 +50,7 @@ @optional_prior_accepted() @values_required @period_required -@post_data_checked_for_required_resolution("market") +@post_data_checked_for_required_resolution("market", "fm1") def post_price_data_response( # noqa C901 unit, generic_asset_name_groups, @@ -80,12 +80,12 @@ def post_price_data_response( # noqa C901 ea = parse_entity_address(market, entity_type="market") except EntityAddressException as eae: return invalid_domain(str(eae)) - market_name = ea["market_name"] + market_id = ea["sensor_id"] # Look for the Market object - market = Market.query.filter(Market.name == market_name).one_or_none() + market = Market.query.filter(Market.id == market_id).one_or_none() if market is None: - return unrecognized_market(market_name) + return unrecognized_market(market_id) elif unit != market.unit: return invalid_unit("%s prices" % market.display_name, [market.unit]) @@ -129,12 +129,12 @@ def post_price_data_response( # noqa C901 @type_accepted("PostWeatherDataRequest") @unit_required -@assets_required("sensor") +@assets_required("weather_sensor") @optional_horizon_accepted() @optional_prior_accepted() @values_required @period_required -@post_data_checked_for_required_resolution("weather_sensor") +@post_data_checked_for_required_resolution("weather_sensor", "fm1") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -225,7 +225,7 @@ def post_weather_data_response( # noqa: C901 @optional_horizon_accepted(ex_post=True) @optional_prior_accepted(ex_post=True) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm1") @as_json def post_meter_data_response( unit, @@ -257,7 +257,7 @@ def post_meter_data_response( @optional_horizon_accepted(ex_post=False) @optional_prior_accepted(ex_post=False) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm1") @as_json def post_prognosis_response( unit, @@ -314,10 +314,10 @@ def post_power_data( # TODO: get asset through util function after refactoring # Parse the entity address try: - connection = parse_entity_address(connection, entity_type="connection") + ea = parse_entity_address(connection, entity_type="connection") except EntityAddressException as eae: return invalid_domain(str(eae)) - asset_id = connection["asset_id"] + asset_id = ea["sensor_id"] # Look for the Asset object if asset_id in user_asset_ids: diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py index 6462e292e..9fbc5071a 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py @@ -3,7 +3,7 @@ from datetime import timedelta from iso8601 import parse_date -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v2_0.tests.utils import ( message_for_post_price_data, @@ -38,7 +38,7 @@ def test_post_price_data_2_0(db, app, post_message): assert post_price_data_response.json["type"] == "PostPriceDataResponse" verify_sensor_data_in_db( - post_message, post_message["values"], db, entity_type="market" + post_message, post_message["values"], db, entity_type="market", fm_scheme="fm1" ) # look for Forecasting jobs in queue @@ -47,7 +47,7 @@ def test_post_price_data_2_0(db, app, post_message): ) # only one market is affected, but two horizons horizons = [timedelta(hours=24), timedelta(hours=48)] jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", fm_scheme="fm1").deserialize(post_message["market"]) for job, horizon in zip(jobs, horizons): assert job.kwargs["horizon"] == horizon assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon @@ -56,12 +56,12 @@ def test_post_price_data_2_0(db, app, post_message): @pytest.mark.parametrize( - "post_message", + "post_message, fm_scheme", [ - message_for_post_prognosis(), + (message_for_post_prognosis(), "fm1"), ], ) -def test_post_prognosis(db, app, post_message): +def test_post_prognosis_2_0(db, app, post_message, fm_scheme): with app.test_client() as client: # post price data auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") @@ -79,5 +79,6 @@ def test_post_prognosis(db, app, post_message): post_message["values"], db, entity_type="connection", + fm_scheme=fm_scheme, swapped_sign=True, ) diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index f1a1054de..a096363a6 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -5,9 +5,10 @@ import pandas as pd import timely_beliefs as tb -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.weather import WeatherSensor, Weather from flexmeasures.data.services.users import find_user_by_email from flexmeasures.api.v1_1.tests.utils import ( @@ -58,6 +59,7 @@ def message_for_post_price_data( duration=duration, invalid_unit=invalid_unit, ) + message["market"] = "ea1.2018-06.localhost:fm1.1" message["horizon"] = duration_isoformat(timedelta(hours=0)) if no_horizon or prior_instead_of_horizon: message.pop("horizon", None) @@ -67,10 +69,18 @@ def message_for_post_price_data( def verify_sensor_data_in_db( - post_message, values, db, entity_type: str, swapped_sign: bool = False + post_message, + values, + db, + entity_type: str, + fm_scheme: str, + swapped_sign: bool = False, ): """util method to verify that sensor data ended up in the database""" - if entity_type == "connection": + if entity_type == "sensor": + sensor_type = Sensor + data_type = TimedBelief + elif entity_type == "connection": sensor_type = Asset data_type = Power elif entity_type == "market": @@ -84,9 +94,9 @@ def verify_sensor_data_in_db( start = parse_datetime(post_message["start"]) end = start + parse_duration(post_message["duration"]) - sensor: Union[Asset, Market, WeatherSensor] = get_generic_asset( - post_message[entity_type], entity_type - ) + sensor: Union[Sensor, Asset, Market, WeatherSensor] = SensorField( + entity_type, fm_scheme + ).deserialize(post_message[entity_type]) resolution = sensor.event_resolution if "horizon" in post_message: horizon = parse_duration(post_message["horizon"]) @@ -133,10 +143,10 @@ def verify_sensor_data_in_db( assert bdf["event_value"].tolist() == values -def message_for_post_prognosis(): +def message_for_post_prognosis(fm_scheme: str = "fm1"): message = { "type": "PostPrognosisRequest", - "connection": "ea1.2018-06.localhost:1:2", + "connection": f"ea1.2018-06.localhost:{fm_scheme}.2", "values": [300, 300, 300, 0, 0, 300], "start": "2021-01-01T00:00:00Z", "duration": "PT1H30M", diff --git a/flexmeasures/data/models/assets.py b/flexmeasures/data/models/assets.py index 68a0a18f3..af7ab4b19 100644 --- a/flexmeasures/data/models/assets.py +++ b/flexmeasures/data/models/assets.py @@ -135,11 +135,19 @@ def power_unit(self) -> float: return self.unit @property - def entity_address(self) -> str: + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" return build_entity_address( - dict(owner_id=self.owner_id, asset_id=self.id), "connection" + dict(owner_id=self.owner_id, asset_id=self.id), + "connection", + fm_scheme="fm0", ) + @property + def entity_address(self) -> str: + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address(dict(sensor_id=self.id), "sensor") + @property def location(self) -> Tuple[float, float]: return self.latitude, self.longitude diff --git a/flexmeasures/data/models/markets.py b/flexmeasures/data/models/markets.py index eb481dffe..dd32431c2 100644 --- a/flexmeasures/data/models/markets.py +++ b/flexmeasures/data/models/markets.py @@ -79,9 +79,17 @@ def __init__(self, **kwargs): if "display_name" not in kwargs: self.display_name = humanize(self.name) + @property + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" + return build_entity_address( + dict(market_name=self.name), "market", fm_scheme="fm0" + ) + @property def entity_address(self) -> str: - return build_entity_address(dict(market_name=self.name), "market") + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address(dict(sensor_id=self.id), "sensor") @property def price_unit(self) -> str: diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 50b404715..c1cf4563d 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -82,7 +82,8 @@ def __init__(self, **kwargs): self.name = self.name.replace(" ", "_").lower() @property - def entity_address(self) -> str: + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" return build_entity_address( dict( weather_sensor_type_name=self.weather_sensor_type_name, @@ -90,6 +91,15 @@ def entity_address(self) -> str: longitude=self.longitude, ), "weather_sensor", + fm_scheme="fm0", + ) + + @property + def entity_address(self) -> str: + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address( + dict(sensor_id=self.id), + "sensor", ) @property diff --git a/flexmeasures/ui/templates/crud/assets.html b/flexmeasures/ui/templates/crud/assets.html index 7324c5c31..a44e932c6 100644 --- a/flexmeasures/ui/templates/crud/assets.html +++ b/flexmeasures/ui/templates/crud/assets.html @@ -21,6 +21,7 @@

All assets

Asset id Owner id Entity address + Old entity address (API v1) {% if user_is_admin %}
@@ -57,6 +58,9 @@

All assets

{{ asset.entity_address }} + + {{ asset.entity_address_fm0 }} + diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 7bbd56c0d..d3350bcd5 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -42,6 +42,8 @@ ADDR_SCHEME = "ea1" +FM1_ADDR_SCHEME = "fm1" +FM0_ADDR_SCHEME = "fm0" class EntityAddressException(Exception): @@ -63,11 +65,17 @@ def get_host() -> str: def build_entity_address( - entity_info: dict, entity_type: str, host: Optional[str] = None + entity_info: dict, + entity_type: str, + host: Optional[str] = None, + fm_scheme: str = FM1_ADDR_SCHEME, ) -> str: """ Build an entity address. + fm1 type entity address should use entity_info["sensor_id"] + todo: implement entity addresses for actuators with entity_info["actuator_id"] (first ensuring globally unique ids across sensors and actuators) + If the host is not given, it is attempted to be taken from the request. entity_info is expected to contain the required fields for the custom string. @@ -83,10 +91,16 @@ def build_field(field: str, required: bool = True): ) if field not in entity_info: return "" - return f":{entity_info[field]}" + return f"{entity_info[field]}:" - if entity_type == "sensor": + if fm_scheme == FM1_ADDR_SCHEME: # and entity_type == "sensor": locally_unique_str = f"{build_field('sensor_id')}" + # elif fm_scheme == FM1_ADDR_SCHEME and entity_type == "actuator": + # locally_unique_str = f"{build_field('actuator_id')}" + elif fm_scheme != FM0_ADDR_SCHEME: + raise EntityAddressException( + f"Unrecognized FlexMeasures scheme for entity addresses: {fm_scheme}" + ) elif entity_type == "connection": locally_unique_str = ( f"{build_field('owner_id', required=False)}{build_field('asset_id')}" @@ -99,12 +113,19 @@ def build_field(field: str, required: bool = True): locally_unique_str = f"{build_field('owner_id', required=False)}{build_field('asset_id')}{build_field('event_id')}{build_field('event_type')}" else: raise EntityAddressException(f"Unrecognized entity type: {entity_type}") - return build_ea_scheme_and_naming_authority(host) + locally_unique_str + return ( + build_ea_scheme_and_naming_authority(host) + + ":" + + fm_scheme + + "." + + locally_unique_str.rstrip(":") + ) def parse_entity_address( # noqa: C901 entity_address: str, entity_type: str, + fm_scheme: str = FM1_ADDR_SCHEME, ) -> dict: """ Parses a generic asset name into an info dict. @@ -112,27 +133,37 @@ def parse_entity_address( # noqa: C901 The entity_address must be a valid type 1 USEF entity address. That is, it must follow the EA1 addressing scheme recommended by USEF. In addition, FlexMeasures expects the identifying string to contain information in - a certain structure. + a certain structure. We distinguish type 0 and type 1 FlexMeasures entity addresses. + + Examples for the fm1 scheme: - For example: + sensor = ea1.2021-01.io.flexmeasures:fm1.42 + sensor = ea1.2021-01.io.flexmeasures:fm1. + connection = ea1.2021-01.io.flexmeasures:fm1. + market = ea1.2021-01.io.flexmeasures:fm1. + weather_station = ea1.2021-01.io.flexmeasures:fm1. + todo: UDI events are not yet modelled in the fm1 scheme, but will probably be ea1.2021-01.io.flexmeasures:fm1. - sensor = ea1.2021-01.io.flexmeasures:42 - sensor = ea1.2021-01.io.flexmeasures: - connection = ea1.2021-01.localhost:40:30 - connection = ea1.2021-01.io.flexmeasures:: - weather_sensor = ea1.2021-01.io.flexmeasures:temperature:52:73.0 - weather_sensor = ea1.2021-01.io.flexmeasures::: - market = ea1.2021-01.io.flexmeasures:epex_da - market = ea1.2021-01.io.flexmeasures: - event = ea1.2021-01.io.flexmeasures:40:30:302:soc - event = ea1.2021-01.io.flexmeasures:::: + Examples for the fm0 scheme: + + connection = ea1.2021-01.localhost:fm0.40:30 + connection = ea1.2021-01.io.flexmeasures:fm0.: + weather_sensor = ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0 + weather_sensor = ea1.2021-01.io.flexmeasures:fm0.:: + market = ea1.2021-01.io.flexmeasures:fm0.epex_da + market = ea1.2021-01.io.flexmeasures:fm0. + event = ea1.2021-01.io.flexmeasures:fm0.40:30:302:soc + event = ea1.2021-01.io.flexmeasures:fm0.::: + + For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. Returns a dictionary with scheme, naming_authority and various other fields, - depending on the entity type (see examples above). + depending on the entity type and FlexMeasures scheme (see examples above). Returns None if entity type is unknown or entity_address is not parseable. We recommend to `return invalid_domain()` in that case. """ - # we can rigidly test the start + + # Check the scheme and naming authority date if not entity_address.startswith(ADDR_SCHEME): raise EntityAddressException( f"A valid type 1 USEF entity address starts with '{ADDR_SCHEME}', please review {entity_address}" @@ -142,16 +173,36 @@ def parse_entity_address( # noqa: C901 raise EntityAddressException( f"After '{ADDR_SCHEME}.', a date specification of the format {date_regex} is expected." ) - # Also the entity type + + # Check the entity type if entity_type not in ("sensor", "connection", "weather_sensor", "market", "event"): raise EntityAddressException(f"Unrecognized entity type: {entity_type}") - if entity_type == "sensor": + def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): + if "fm_scheme" not in ea: + # Backwards compatibility: assume fm0 if fm_scheme is not specified + ea["fm_scheme"] = FM0_ADDR_SCHEME + scheme = ea["scheme"] + naming_authority = ea["naming_authority"] + if ea["fm_scheme"] != fm_scheme: + raise EntityAddressException( + f"A valid type {fm_scheme[2:]} FlexMeasures entity address starts with '{scheme}.{naming_authority}:{fm_scheme}', please review {entity_address}" + ) + + if fm_scheme == FM1_ADDR_SCHEME: + + # Check the FlexMeasures scheme + if entity_address.split(":")[1][: len(fm_scheme) + 1] != FM1_ADDR_SCHEME + ".": + raise EntityAddressException( + f"A valid type {fm_scheme[2:]} FlexMeasures entity address starts with '{build_ea_scheme_and_naming_authority(get_host())}:{fm_scheme}.', please review {entity_address}" + ) + match = re.search( r"^" r"(?P.+)\." fr"(?P{date_regex}\.[^:]+)" # everything until the colon (no port) r":" + r"((?P.+)\.)" r"(?P\d+)" r"$", entity_address, @@ -160,15 +211,24 @@ def parse_entity_address( # noqa: C901 value_types = { "scheme": str, "naming_authority": str, + "fm_scheme": str, "sensor_id": int, } - return _typed_regex_results(match, value_types) + else: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + elif fm_scheme != FM0_ADDR_SCHEME: + raise EntityAddressException( + f"Unrecognized FlexMeasures scheme for entity addresses: {fm_scheme}" + ) elif entity_type == "connection": match = re.search( r"^" r"(?P.+)\." fr"(?P{date_regex}\.[^:]+)" # everything until the colon (no port) r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"((?P\d+):)*" # owner id is optional r"(?P\d+)" r"$", @@ -181,7 +241,10 @@ def parse_entity_address( # noqa: C901 "owner_id": int, "asset_id": int, } - return _typed_regex_results(match, value_types) + else: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) elif entity_type == "weather_sensor": match = re.search( r"^" @@ -189,6 +252,7 @@ def parse_entity_address( # noqa: C901 r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"(?=[a-zA-Z])(?P[\w]+)" # should start with at least one letter r":" r"(?P\-?\d+(\.\d+)?)" @@ -205,7 +269,10 @@ def parse_entity_address( # noqa: C901 "latitude": float, "longitude": float, } - return _typed_regex_results(match, value_types) + else: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) elif entity_type == "market": match = re.search( r"^" @@ -213,13 +280,17 @@ def parse_entity_address( # noqa: C901 r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"(?=[a-zA-Z])(?P[\w]+)" # should start with at least one letter r"$", entity_address, ) if match: value_types = {"scheme": str, "naming_authority": str, "market_name": str} - return _typed_regex_results(match, value_types) + else: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) elif entity_type == "event": match = re.search( r"^" @@ -227,6 +298,7 @@ def parse_entity_address( # noqa: C901 r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"((?P\d+):)*" # owner id is optional r"(?P\d+)" r":" @@ -245,10 +317,17 @@ def parse_entity_address( # noqa: C901 "event_id": int, "event_type": str, } - return _typed_regex_results(match, value_types) + else: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + else: + # Finally, we simply raise without precise information what went wrong + raise EntityAddressException(f"Could not parse {entity_address}.") - # Finally, we simply raise without precise information what went wrong - raise EntityAddressException(f"Could not parse {entity_address}.") + ea = _typed_regex_results(match, value_types) + validate_ea_for_fm_scheme(ea, fm_scheme) + return ea def build_ea_scheme_and_naming_authority( diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index 85d0357fb..24a45600c 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -11,37 +11,56 @@ @pytest.mark.parametrize( - "info, entity_type, host, exp_result", + "info, entity_type, host, fm_scheme, exp_result", [ ( dict(sensor_id=42), "sensor", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:42", + "fm1", + "ea1.2021-01.io.flexmeasures:fm1.42", ), ( dict(asset_id=42), "sensor", "flexmeasures.io", + "fm1", + "required field 'sensor_id'", + ), + ( + dict(sensor_id=40), + "connection", + "flexmeasures.io", + "fm1", + "ea1.2021-01.io.flexmeasures:fm1.40", + ), + ( + dict(asset_id=40), + "connection", + "flexmeasures.io", + "fm1", "required field 'sensor_id'", ), ( dict(owner_id=3, asset_id=40), "connection", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:3:40", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.3:40", ), ( dict(owner_id=3), "connection", "flexmeasures.io", + "fm0", "required field 'asset_id'", ), ( dict(owner_id=40, asset_id=30), "connection", "localhost:5000", - f"ea1.{get_first_day_of_next_month().strftime('%Y-%m')}.localhost:40:30", + "fm0", + f"ea1.{get_first_day_of_next_month().strftime('%Y-%m')}.localhost:fm0.40:30", ), ( dict( @@ -51,13 +70,15 @@ ), "weather_sensor", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:temperature:52:73.0", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0", ), ( dict(market_name="epex_da"), "market", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:epex_da", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.epex_da", ), ( dict( @@ -68,12 +89,13 @@ ), "event", "http://staging.flexmeasures.io:4444", - "ea1.2022-09.io.flexmeasures.staging:40:30:302:soc", + "fm0", + "ea1.2022-09.io.flexmeasures.staging:fm0.40:30:302:soc", ), ], ) def test_build_entity_address( - app, info: dict, entity_type: str, host: str, exp_result: str + app, info: dict, entity_type: str, host: str, fm_scheme: str, exp_result: str ): with app.app_context(): app.config["FLEXMEASURES_HOSTS_AND_AUTH_START"] = { @@ -81,10 +103,12 @@ def test_build_entity_address( "staging.flexmeasures.io": "2022-09", } if exp_result.startswith("ea1"): - assert build_entity_address(info, entity_type, host) == exp_result + assert ( + build_entity_address(info, entity_type, host, fm_scheme) == exp_result + ) else: with pytest.raises(EntityAddressException, match=exp_result): - build_entity_address(info, entity_type, host) == exp_result + build_entity_address(info, entity_type, host, fm_scheme) == exp_result @pytest.mark.parametrize( @@ -163,9 +187,13 @@ def test_build_entity_address( def test_parse_entity_address(entity_type, entity_address, exp_result): if isinstance(exp_result, str): # this means we expect an exception with pytest.raises(EntityAddressException, match=exp_result): - parse_entity_address(entity_address, entity_type=entity_type) + parse_entity_address( + entity_address, entity_type=entity_type, fm_scheme="fm0" + ) else: - res = parse_entity_address(entity_address, entity_type=entity_type) + res = parse_entity_address( + entity_address, entity_type=entity_type, fm_scheme="fm0" + ) assert res["scheme"] == "ea1" assert res["naming_authority"] == exp_result["naming_authority"] if entity_type in ("connection", "event"): From 2a55a79526ab631b8dbf1544db7cd7e5de86fc58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 19 May 2021 10:52:59 +0200 Subject: [PATCH 18/21] add docstrings to our API documentation, more explicit handling of no regex matches in parsing ea addresses --- Makefile | 4 +- documentation/api/introduction.rst | 64 ++++++++---- flexmeasures/utils/entity_address_utils.py | 111 ++++++++------------- 3 files changed, 86 insertions(+), 93 deletions(-) diff --git a/Makefile b/Makefile index dd8797f93..e071ae945 100644 --- a/Makefile +++ b/Makefile @@ -15,13 +15,13 @@ test: # ---- Documentation --- update-docs: - pip3 install sphinx sphinx-rtd-theme sphinxcontrib.httpdomain + pip3 install sphinx==3.5.4 sphinxcontrib.httpdomain # sphinx4 is not supported yet by sphinx-contrib/httpdomain, see https://github.com/sphinx-contrib/httpdomain/issues/46 cd documentation; make clean; make html; cd .. update-docs-pdf: @echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)" @echo "NOTE: Currently, the docs require some pictures which are not in the git repo atm. Ask the devs." - pip3 install sphinx sphinxcontrib.httpdomain + pip3 install sphinx sphinx-rtd-theme sphinxcontrib.httpdomain cd documentation; make clean; make latexpdf; make latexpdf; cd .. # make latexpdf can require two passes # ---- Installation --- diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 117113d51..f5d9429d9 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -129,8 +129,8 @@ Throughout this document, keys are written in singular if a single value is list The API, however, does not distinguish between singular and plural key notation. -Connections -^^^^^^^^^^^ +Connections and entity addresses +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Connections are end points of the grid at which an asset is located. Connections should be identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], @@ -160,36 +160,62 @@ The owner ID is optional. Both the owner ID and the asset ID, as well as the ful https://company.flexmeasures.io/assets +Entity address structure +"""""""""""""""""""""""""" Some deeper explanations about an entity address: - "ea1" is a constant, indicating this is a type 1 USEF entity address - The date code "must be a date during which the naming authority owned the domain name used in this format, and should be the first month in which the domain name was owned by this naming authority at 00:01 GMT of the first day of the month. - The reversed domain name is taken from the naming authority (person or organization) creating this entity address - The locally unique string can be used for local purposes, and FlexMeasures uses it to identify the resource (more information in parse_entity_address). + Fields in the locally unique string are separated by colons, see for other examples + IETF RFC 3721, page 6 [3]. While [2] says it's possible to use dashes, dots or colons as separators, we might use dashes and dots in + latitude/longitude coordinates of sensors, so we settle on colons. + [1] https://www.usef.energy/app/uploads/2020/01/USEF-Flex-Trading-Protocol-Specifications-1.01.pdf [2] https://tools.ietf.org/html/rfc3720 +[3] https://tools.ietf.org/html/rfc3721 -Notation for simulation -""""""""""""""""""""""" -For version 1 of the API, the following simplified addressing scheme may be used: +Types of asset identifications used in FlexMeasures +"""""""""""""""""""""""""""""""""""""""""" -.. code-block:: json +FlexMeasures expects the locally unique string string to contain information in +a certain structure. We distinguish type ``fm0`` and type ``fm1`` FlexMeasures entity addresses. - { - "connection": ":" - } +The ``fm0`` scheme is the original scheme. It identifies connected assets, sensors and markets with a combined key of type and ID. -or even simpler: +Examples for the fm0 scheme: -.. code-block:: json +- connection = ea1.2021-01.localhost:fm0.40:30 +- connection = ea1.2021-01.io.flexmeasures:fm0.: +- weather_sensor = ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0 +- weather_sensor = ea1.2021-01.io.flexmeasures:fm0.:: +- market = ea1.2021-01.io.flexmeasures:fm0.epex_da +- market = ea1.2021-01.io.flexmeasures:fm0. +- event = ea1.2021-01.io.flexmeasures:fm0.40:30:302:soc +- event = ea1.2021-01.io.flexmeasures:fm0.::: + +This scheme is explicit but also a little cumbersome to use, as one needs to look up the type or even owner (for assets), and weather sensors are identified by coordinates. +For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. + + +The ``fm1`` scheme is the latest version, currently under development. It works with the database structure +we are developing in the background, where all connected sensors have unique IDs. This makes it more straightforward, if less explicit. + +Examples for the fm1 scheme: + +- sensor = ea1.2021-01.io.flexmeasures:fm1.42 +- sensor = ea1.2021-01.io.flexmeasures:fm1. +- connection = ea1.2021-01.io.flexmeasures:fm1. +- market = ea1.2021-01.io.flexmeasures:fm1. +- weather_station = ea1.2021-01.io.flexmeasures:fm1. + +.. todo:: UDI events are not yet modelled in the fm1 scheme, but will probably be ea1.2021-01.io.flexmeasures:fm1. - { - "connection": "" - } Groups ^^^^^^ @@ -203,8 +229,8 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups "groups": [ { "connections": [ - "CS 1", - "CS 2" + "ea1.2021-02.io.flexmeasures.company:30:71", + "ea1.2021-02.io.flexmeasures.company:30:72" ], "values": [ 306.66, @@ -216,7 +242,7 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups ] }, { - "connection": "CS 3", + "connection": "ea1.2021-02.io.flexmeasures.company:30:73" "values": [ 306.66, 0, @@ -238,8 +264,8 @@ In case of a single group of connections, the message may be flattened to: { "connections": [ - "CS 1", - "CS 2" + "ea1.2021-02.io.flexmeasures.company:30:71", + "ea1.2021-02.io.flexmeasures.company:30:72" ], "values": [ 306.66, diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index d3350bcd5..3f28abed4 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -11,33 +11,10 @@ """ -Functionality to support parsing and building USEF's EA1 addressing scheme [1], -which is mostly taken from IETF RFC 3720 [2]: - -This is the complete structure of an EA1 address: - -ea1.{date code}.{reversed domain name}:{locally unique string} - -for example "ea1.2021-01.io.flexmeasures.company:sensor14" - -- "ea1" is a constant, indicating this is a type 1 USEF entity address -- The date code "must be a date during which the naming authority owned - the domain name used in this format, and should be the first month in which the domain name was - owned by this naming authority at 00:01 GMT of the first day of the month. -- The reversed domain name is taken from the naming authority - (person or organization) creating this entity address -- The locally unique string can be used for local purposes, and FlexMeasures - uses it to identify the resource (more information in parse_entity_address). - Fields in the locally unique string are separated by colons, see for other examples - IETF RFC 3721, page 6 [3]. - ([2] says it's possible to use dashes, dots or colons ― dashes and dots might come up in - latitude/longitude coordinates of sensors) - -TODO: This needs to be in the FlexMeasures documentation. +Functionality to support parsing and building Entity Addresses as defined by USEF [1]. +See our documentation for more details. [1] https://www.usef.energy/app/uploads/2020/01/USEF-Flex-Trading-Protocol-Specifications-1.01.pdf -[2] https://tools.ietf.org/html/rfc3720 -[3] https://tools.ietf.org/html/rfc3721 """ @@ -130,10 +107,10 @@ def parse_entity_address( # noqa: C901 """ Parses a generic asset name into an info dict. - The entity_address must be a valid type 1 USEF entity address. - That is, it must follow the EA1 addressing scheme recommended by USEF. - In addition, FlexMeasures expects the identifying string to contain information in - a certain structure. We distinguish type 0 and type 1 FlexMeasures entity addresses. + Returns a dictionary with scheme, naming_authority and various other fields, + depending on the entity type and FlexMeasures scheme (see examples above). + Returns None if entity type is unknown or entity_address is not parse-able. + We recommend to `return invalid_domain()` in that case. Examples for the fm1 scheme: @@ -156,11 +133,6 @@ def parse_entity_address( # noqa: C901 event = ea1.2021-01.io.flexmeasures:fm0.::: For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. - - Returns a dictionary with scheme, naming_authority and various other fields, - depending on the entity type and FlexMeasures scheme (see examples above). - Returns None if entity type is unknown or entity_address is not parseable. - We recommend to `return invalid_domain()` in that case. """ # Check the scheme and naming authority date @@ -207,17 +179,16 @@ def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "fm_scheme": str, - "sensor_id": int, - } - else: + if match is None: raise EntityAddressException( f"Could not parse {entity_type} {entity_address}." ) + value_types = { + "scheme": str, + "naming_authority": str, + "fm_scheme": str, + "sensor_id": int, + } elif fm_scheme != FM0_ADDR_SCHEME: raise EntityAddressException( f"Unrecognized FlexMeasures scheme for entity addresses: {fm_scheme}" @@ -234,17 +205,16 @@ def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "owner_id": int, - "asset_id": int, - } - else: + if match is None: raise EntityAddressException( f"Could not parse {entity_type} {entity_address}." ) + value_types = { + "scheme": str, + "naming_authority": str, + "owner_id": int, + "asset_id": int, + } elif entity_type == "weather_sensor": match = re.search( r"^" @@ -261,18 +231,17 @@ def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "weather_sensor_type_name": str, - "latitude": float, - "longitude": float, - } - else: + if match is None: raise EntityAddressException( f"Could not parse {entity_type} {entity_address}." ) + value_types = { + "scheme": str, + "naming_authority": str, + "weather_sensor_type_name": str, + "latitude": float, + "longitude": float, + } elif entity_type == "market": match = re.search( r"^" @@ -285,12 +254,11 @@ def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): r"$", entity_address, ) - if match: - value_types = {"scheme": str, "naming_authority": str, "market_name": str} - else: + if match is None: raise EntityAddressException( f"Could not parse {entity_type} {entity_address}." ) + value_types = {"scheme": str, "naming_authority": str, "market_name": str} elif entity_type == "event": match = re.search( r"^" @@ -308,19 +276,18 @@ def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "owner_id": int, - "asset_id": int, - "event_id": int, - "event_type": str, - } - else: + if match is None: raise EntityAddressException( f"Could not parse {entity_type} {entity_address}." ) + value_types = { + "scheme": str, + "naming_authority": str, + "owner_id": int, + "asset_id": int, + "event_id": int, + "event_type": str, + } else: # Finally, we simply raise without precise information what went wrong raise EntityAddressException(f"Could not parse {entity_address}.") From f146abcd3328c3d746580baab3272b537b0a9ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 19 May 2021 11:05:54 +0200 Subject: [PATCH 19/21] also test parsing a fm1 type address --- flexmeasures/utils/tests/test_entity_address_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index 24a45600c..b13a249bf 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -182,6 +182,16 @@ def test_build_entity_address( event_id=302, ), ), + ( + "connection", + "ea1.2018-06.io.flexmeasures.staging:fm1.30", + dict( + naming_authority="2018-06.io.flexmeasures.staging", + asset_id=30, + owner_id=None, + event_type="connection", + ), + ), ], ) def test_parse_entity_address(entity_type, entity_address, exp_result): From 1594c126a53470e01ada830079e00a09537626d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 21 May 2021 09:20:11 +0200 Subject: [PATCH 20/21] review comments: typos, nomenclature --- documentation/api/introduction.rst | 4 ++-- flexmeasures/api/v2_0/tests/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index e422d8eab..912a20f60 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -186,7 +186,7 @@ Types of asset identifications used in FlexMeasures FlexMeasures expects the locally unique string string to contain information in a certain structure. We distinguish type ``fm0`` and type ``fm1`` FlexMeasures entity addresses. -The ``fm0`` scheme is the original scheme. It identifies connected assets, sensors and markets with a combined key of type and ID. +The ``fm0`` scheme is the original scheme. It identifies connected assets, weather stations, markets and UDI events in different ways. Examples for the fm0 scheme: @@ -204,7 +204,7 @@ For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. The ``fm1`` scheme is the latest version, currently under development. It works with the database structure -we are developing in the background, where all connected sensors have unique IDs. This makes it more straightforward, if less explicit. +we are developing in the background, where all connected sensors have unique IDs. This makes it more straightforward (the scheme works the same way for all types of sensors), if less explicit. Examples for the fm1 scheme: diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index 4bc5616bd..215b6d0e8 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -146,7 +146,7 @@ def verify_sensor_data_in_db( def message_for_post_prognosis(fm_scheme: str = "fm1"): """ - Posting prognosis for a wind mill's production. + Posting prognosis for a wind turbine's production. """ message = { "type": "PostPrognosisRequest", From 580fe7c95c75383c21300d5d179437835b8899cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 28 May 2021 11:31:53 +0200 Subject: [PATCH 21/21] implement review comments --- documentation/api/change_log.rst | 10 +++++++++- documentation/api/introduction.rst | 19 ++++++++++--------- documentation/changelog.rst | 1 + 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 4bc6712c4..271756247 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,14 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This also reflects in the URL, allowing developers to upgrade at their own pace. +v2.0-3 | 2021-05-XX +""""""""""""""""" + +- Updated all entity addresses in documentation according to the fm0 scheme, preserving backwards compatibility. +- Introduced the fm1 scheme for entity addresses for connections, markets, weather sensors and sensors. + + + v2.0-2 | 2021-04-02 """"""""""""""""" @@ -31,7 +39,7 @@ v2.0-0 | 2020-11-14 - REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). -v1.3.9 | 2021-04-XX +v1.3.9 | 2021-04-21 """"""""""""""""" *Affects all versions since v1.0*. diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 912a20f60..4a55e5073 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -149,11 +149,12 @@ Here is a full example for a FlexMeasures connection address: .. code-block:: json { - "connection": "ea1.2021-02.io.flexmeasures.company:30:73" + "connection": "ea1.2021-02.io.flexmeasures.company:fm0.30:73" } -where FlexMeasures runs at `company.flexmeasures.io` and the owner ID is 30 and the asset ID is 73. -The owner ID is optional. Both the owner ID and the asset ID, as well as the full entity address can be obtained on the asset's listing after logging in: +where FlexMeasures runs at `company.flexmeasures.io` (which the current domain owner started using in February 2021), and the locally unique string is of scheme `fm0` (see below) and the asset ID is 73. The asset's owner ID is 30, but this part is optional. + +Both the owner ID and the asset ID, as well as the full entity address can be obtained on the asset's listing: .. code-block:: html @@ -167,7 +168,7 @@ Some deeper explanations about an entity address: - "ea1" is a constant, indicating this is a type 1 USEF entity address - The date code "must be a date during which the naming authority owned the domain name used in this format, and should be the first month in which the domain name was owned by this naming authority at 00:01 GMT of the first day of the month. - The reversed domain name is taken from the naming authority (person or organization) creating this entity address -- The locally unique string can be used for local purposes, and FlexMeasures uses it to identify the resource (more information in parse_entity_address). +- The locally unique string can be used for local purposes, and FlexMeasures uses it to identify the resource. Fields in the locally unique string are separated by colons, see for other examples IETF RFC 3721, page 6 [3]. While [2] says it's possible to use dashes, dots or colons as separators, we might use dashes and dots in latitude/longitude coordinates of sensors, so we settle on colons. @@ -229,8 +230,8 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups "groups": [ { "connections": [ - "ea1.2021-02.io.flexmeasures.company:30:71", - "ea1.2021-02.io.flexmeasures.company:30:72" + "ea1.2021-02.io.flexmeasures.company:fm0.30:71", + "ea1.2021-02.io.flexmeasures.company:fm0.30:72" ], "values": [ 306.66, @@ -242,7 +243,7 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups ] }, { - "connection": "ea1.2021-02.io.flexmeasures.company:30:73" + "connection": "ea1.2021-02.io.flexmeasures.company:fm0.30:73" "values": [ 306.66, 0, @@ -264,8 +265,8 @@ In case of a single group of connections, the message may be flattened to: { "connections": [ - "ea1.2021-02.io.flexmeasures.company:30:71", - "ea1.2021-02.io.flexmeasures.company:30:72" + "ea1.2021-02.io.flexmeasures.company:fm0.30:71", + "ea1.2021-02.io.flexmeasures.company:fm0.30:72" ], "values": [ 306.66, diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c665c67c3..ca3f9369b 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -22,6 +22,7 @@ Infrastructure / Support ---------------------- * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] * For weather forecasts, switch from Dark Sky (closed from Aug 1, 2021) to OpenWeatherMap API [see `PR #113 `_] +* Entity address improvements: add new id-based `fm1` scheme, better documentation and more validation support of entity addresses [see `PR #81 `_] * Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run [see `PR #115 `_] * Let CLI package and plugins use Marshmallow Field definitions [see `PR #125 `_]