diff --git a/Makefile b/Makefile index ce465bc92..e071ae945 100644 --- a/Makefile +++ b/Makefile @@ -15,13 +15,13 @@ test: # ---- Documentation --- update-docs: - pip3 install sphinx==3.5.4 sphinxcontrib.httpdomain # sphinx4 is not supported yet by sphinx-contrib/httpdomain, see https://github.com/sphinx-contrib/httpdomain/issues/46 + pip3 install sphinx==3.5.4 sphinxcontrib.httpdomain # sphinx4 is not supported yet by sphinx-contrib/httpdomain, see https://github.com/sphinx-contrib/httpdomain/issues/46 cd documentation; make clean; make html; cd .. update-docs-pdf: @echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)" @echo "NOTE: Currently, the docs require some pictures which are not in the git repo atm. Ask the devs." - pip3 install sphinx sphinxcontrib.httpdomain + pip3 install sphinx sphinx-rtd-theme sphinxcontrib.httpdomain cd documentation; make clean; make latexpdf; make latexpdf; cd .. # make latexpdf can require two passes # ---- Installation --- diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 4bc6712c4..271756247 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,14 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This also reflects in the URL, allowing developers to upgrade at their own pace. +v2.0-3 | 2021-05-XX +""""""""""""""""" + +- Updated all entity addresses in documentation according to the fm0 scheme, preserving backwards compatibility. +- Introduced the fm1 scheme for entity addresses for connections, markets, weather sensors and sensors. + + + v2.0-2 | 2021-04-02 """"""""""""""""" @@ -31,7 +39,7 @@ v2.0-0 | 2020-11-14 - REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/` (GET, PATCH, DELETE). -v1.3.9 | 2021-04-XX +v1.3.9 | 2021-04-21 """"""""""""""""" *Affects all versions since v1.0*. diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 938541808..59d547a93 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -179,8 +179,8 @@ Throughout this document, keys are written in singular if a single value is list The API, however, does not distinguish between singular and plural key notation. -Connections -^^^^^^^^^^^ +Connections and entity addresses +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Connections are end points of the grid at which an asset is located. Connections should be identified with an entity address following the EA1 addressing scheme prescribed by USEF[1], @@ -199,47 +199,74 @@ Here is a full example for a FlexMeasures connection address: .. code-block:: json { - "connection": "ea1.2021-02.io.flexmeasures.company:30:73" + "connection": "ea1.2021-02.io.flexmeasures.company:fm0.30:73" } -where FlexMeasures runs at `company.flexmeasures.io` and the owner ID is 30 and the asset ID is 73. -The owner ID is optional. Both the owner ID and the asset ID, as well as the full entity address can be obtained on the asset's listing after logging in: +where FlexMeasures runs at `company.flexmeasures.io` (which the current domain owner started using in February 2021), and the locally unique string is of scheme `fm0` (see below) and the asset ID is 73. The asset's owner ID is 30, but this part is optional. + +Both the owner ID and the asset ID, as well as the full entity address can be obtained on the asset's listing: .. code-block:: html https://company.flexmeasures.io/assets +Entity address structure +"""""""""""""""""""""""""" Some deeper explanations about an entity address: - "ea1" is a constant, indicating this is a type 1 USEF entity address - The date code "must be a date during which the naming authority owned the domain name used in this format, and should be the first month in which the domain name was owned by this naming authority at 00:01 GMT of the first day of the month. - The reversed domain name is taken from the naming authority (person or organization) creating this entity address -- The locally unique string can be used for local purposes, and FlexMeasures uses it to identify the resource (more information in parse_entity_address). +- The locally unique string can be used for local purposes, and FlexMeasures uses it to identify the resource. + Fields in the locally unique string are separated by colons, see for other examples + IETF RFC 3721, page 6 [3]. While [2] says it's possible to use dashes, dots or colons as separators, we might use dashes and dots in + latitude/longitude coordinates of sensors, so we settle on colons. + [1] https://www.usef.energy/app/uploads/2020/01/USEF-Flex-Trading-Protocol-Specifications-1.01.pdf [2] https://tools.ietf.org/html/rfc3720 +[3] https://tools.ietf.org/html/rfc3721 -Notation for simulation -""""""""""""""""""""""" -For version 1 of the API, the following simplified addressing scheme may be used: +Types of asset identifications used in FlexMeasures +"""""""""""""""""""""""""""""""""""""""""" -.. code-block:: json +FlexMeasures expects the locally unique string string to contain information in +a certain structure. We distinguish type ``fm0`` and type ``fm1`` FlexMeasures entity addresses. - { - "connection": ":" - } +The ``fm0`` scheme is the original scheme. It identifies connected assets, weather stations, markets and UDI events in different ways. -or even simpler: +Examples for the fm0 scheme: -.. code-block:: json +- connection = ea1.2021-01.localhost:fm0.40:30 +- connection = ea1.2021-01.io.flexmeasures:fm0.: +- weather_sensor = ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0 +- weather_sensor = ea1.2021-01.io.flexmeasures:fm0.:: +- market = ea1.2021-01.io.flexmeasures:fm0.epex_da +- market = ea1.2021-01.io.flexmeasures:fm0. +- event = ea1.2021-01.io.flexmeasures:fm0.40:30:302:soc +- event = ea1.2021-01.io.flexmeasures:fm0.::: + +This scheme is explicit but also a little cumbersome to use, as one needs to look up the type or even owner (for assets), and weather sensors are identified by coordinates. +For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. + + +The ``fm1`` scheme is the latest version, currently under development. It works with the database structure +we are developing in the background, where all connected sensors have unique IDs. This makes it more straightforward (the scheme works the same way for all types of sensors), if less explicit. + +Examples for the fm1 scheme: + +- sensor = ea1.2021-01.io.flexmeasures:fm1.42 +- sensor = ea1.2021-01.io.flexmeasures:fm1. +- connection = ea1.2021-01.io.flexmeasures:fm1. +- market = ea1.2021-01.io.flexmeasures:fm1. +- weather_station = ea1.2021-01.io.flexmeasures:fm1. + +.. todo:: UDI events are not yet modelled in the fm1 scheme, but will probably be ea1.2021-01.io.flexmeasures:fm1. - { - "connection": "" - } Groups ^^^^^^ @@ -253,8 +280,8 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups "groups": [ { "connections": [ - "CS 1", - "CS 2" + "ea1.2021-02.io.flexmeasures.company:fm0.30:71", + "ea1.2021-02.io.flexmeasures.company:fm0.30:72" ], "values": [ 306.66, @@ -266,7 +293,7 @@ When the attributes "start", "duration" and "unit" are stated outside of "groups ] }, { - "connection": "CS 3", + "connection": "ea1.2021-02.io.flexmeasures.company:fm0.30:73" "values": [ 306.66, 0, @@ -288,8 +315,8 @@ In case of a single group of connections, the message may be flattened to: { "connections": [ - "CS 1", - "CS 2" + "ea1.2021-02.io.flexmeasures.company:fm0.30:71", + "ea1.2021-02.io.flexmeasures.company:fm0.30:72" ], "values": [ 306.66, diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c61cb8ee1..fda8c4356 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -22,6 +22,7 @@ Infrastructure / Support ---------------------- * Add tutorials on how to add and read data from FlexMeasures via its API [see `PR #130 `_] * For weather forecasts, switch from Dark Sky (closed from Aug 1, 2021) to OpenWeatherMap API [see `PR #113 `_] +* Entity address improvements: add new id-based `fm1` scheme, better documentation and more validation support of entity addresses [see `PR #81 `_] * Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run [see `PR #115 `_] * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 `_] * Let CLI package and plugins use Marshmallow Field definitions [see `PR #125 `_] diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py new file mode 100644 index 000000000..4989b5fb4 --- /dev/null +++ b/flexmeasures/api/common/schemas/sensors.py @@ -0,0 +1,135 @@ +from typing import Union + +from marshmallow import fields + +from flexmeasures.api import FMValidationError +from flexmeasures.api.common.utils.api_utils import get_weather_sensor_by +from flexmeasures.utils.entity_address_utils import ( + parse_entity_address, + EntityAddressException, +) +from flexmeasures.data.models.assets import Asset +from flexmeasures.data.models.markets import Market +from flexmeasures.data.models.weather import WeatherSensor +from flexmeasures.data.models.time_series import Sensor + + +class EntityAddressValidationError(FMValidationError): + status = "INVALID_DOMAIN" # USEF error status + + +class SensorField(fields.Str): + """Field that de-serializes to a Sensor, Asset, Market or WeatherSensor + and serializes back to an entity address (string).""" + + # todo: when Actuators also get an entity address, refactor this class to EntityField, + # where an Entity represents anything with an entity address: we currently foresee Sensors and Actuators + + def __init__( + self, + entity_type: str, + fm_scheme: str, + *args, + **kwargs, + ): + """ + :param entity_type: "sensor", "connection", "market" or "weather_sensor" + :param fm_scheme: "fm0" or "fm1" + """ + self.entity_type = entity_type + self.fm_scheme = fm_scheme + super().__init__(*args, **kwargs) + + def _deserialize( # noqa: C901 todo: the noqa can probably be removed after refactoring Asset/Market/WeatherSensor to Sensor + self, value, attr, obj, **kwargs + ) -> Union[Sensor, Asset, Market, WeatherSensor]: + """De-serialize to a Sensor, Asset, Market or WeatherSensor.""" + # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case + try: + ea = parse_entity_address(value, self.entity_type, self.fm_scheme) + if self.fm_scheme == "fm0": + if self.entity_type == "connection": + asset = Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() + if asset is not None: + return asset + else: + raise EntityAddressValidationError( + f"Asset with entity address {value} doesn't exist." + ) + elif self.entity_type == "market": + market = Market.query.filter( + Market.name == ea["market_name"] + ).one_or_none() + if market is not None: + return market + else: + raise EntityAddressValidationError( + f"Market with entity address {value} doesn't exist." + ) + elif self.entity_type == "weather_sensor": + weather_sensor = get_weather_sensor_by( + ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"] + ) + if weather_sensor is not None and isinstance( + weather_sensor, WeatherSensor + ): + return weather_sensor + else: + raise EntityAddressValidationError( + f"Weather sensor with entity address {value} doesn't exist." + ) + else: + if self.entity_type == "sensor": + sensor = Sensor.query.filter( + Sensor.id == ea["sensor_id"] + ).one_or_none() + if sensor is not None: + return sensor + else: + raise EntityAddressValidationError( + f"Sensor with entity address {value} doesn't exist." + ) + elif self.entity_type == "connection": + asset = Asset.query.filter( + Asset.id == ea["sensor_id"] + ).one_or_none() + if asset is not None: + return asset + else: + raise EntityAddressValidationError( + f"Asset with entity address {value} doesn't exist." + ) + elif self.entity_type == "market": + market = Market.query.filter( + Market.id == ea["sensor_id"] + ).one_or_none() + if market is not None: + return market + else: + raise EntityAddressValidationError( + f"Market with entity address {value} doesn't exist." + ) + elif self.entity_type == "weather_sensor": + weather_sensor = WeatherSensor.query.filter( + WeatherSensor.id == ea["sensor_id"] + ).one_or_none() + if weather_sensor is not None and isinstance( + weather_sensor, WeatherSensor + ): + return weather_sensor + else: + raise EntityAddressValidationError( + f"Weather sensor with entity address {value} doesn't exist." + ) + except EntityAddressException as eae: + raise EntityAddressValidationError(str(eae)) + return NotImplemented + + def _serialize( + self, value: Union[Sensor, Asset, Market, WeatherSensor], attr, data, **kwargs + ): + """Serialize to an entity address.""" + if self.fm_scheme == "fm0": + return value.entity_address_fm0 + else: + return value.entity_address diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py new file mode 100644 index 000000000..829daf91b --- /dev/null +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -0,0 +1,104 @@ +import pytest + +from flexmeasures.api.common.schemas.sensors import ( + SensorField, + EntityAddressValidationError, +) +from flexmeasures.utils.entity_address_utils import build_entity_address + + +@pytest.mark.parametrize( + "entity_address, entity_type, fm_scheme, exp_deserialization_name", + [ + ( + build_entity_address(dict(sensor_id=1), "sensor"), + "sensor", + "fm1", + "my daughter's height", + ), + ( + build_entity_address( + dict(market_name="epex_da"), "market", fm_scheme="fm0" + ), + "market", + "fm0", + "epex_da", + ), + ( + build_entity_address( + dict(owner_id=1, asset_id=4), "connection", fm_scheme="fm0" + ), + "connection", + "fm0", + "Test battery with no known prices", + ), + ( + build_entity_address( + dict( + weather_sensor_type_name="temperature", + latitude=33.4843866, + longitude=126.0, + ), + "weather_sensor", + fm_scheme="fm0", + ), + "weather_sensor", + "fm0", + "temperature_sensor", + ), + ], +) +def test_sensor_field_straightforward( + add_sensors, + setup_markets, + add_battery_assets, + add_weather_sensors, + entity_address, + entity_type, + fm_scheme, + exp_deserialization_name, +): + """Testing straightforward cases""" + sf = SensorField(entity_type, fm_scheme) + deser = sf.deserialize(entity_address, None, None) + assert deser.name == exp_deserialization_name + assert sf.serialize(entity_type, {entity_type: deser}) == entity_address + + +@pytest.mark.parametrize( + "entity_address, entity_type, fm_scheme, error_msg", + [ + ( + "ea1.2021-01.io.flexmeasures:some.weird:identifier%that^is*not)used", + "market", + "fm0", + "Could not parse", + ), + ( + "ea1.2021-01.io.flexmeasures:fm1.some.weird:identifier%that^is*not)used", + "market", + "fm1", + "Could not parse", + ), + ( + build_entity_address( + dict(market_name="non_existing_market"), "market", fm_scheme="fm0" + ), + "market", + "fm0", + "doesn't exist", + ), + ( + build_entity_address(dict(sensor_id=-1), "sensor", fm_scheme="fm1"), + "market", + "fm1", + "Could not parse", + ), + ("ea1.2021-13.io.flexmeasures:fm1.9", "sensor", "fm1", "date specification"), + ], +) +def test_sensor_field_invalid(entity_address, entity_type, fm_scheme, error_msg): + sf = SensorField(entity_type, fm_scheme) + with pytest.raises(EntityAddressValidationError) as ve: + sf.deserialize(entity_address, None, None) + assert error_msg in str(ve) diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index ac729707d..1bebc73fe 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -12,10 +12,9 @@ from flexmeasures.data import db from flexmeasures.data.models.assets import Asset, Power -from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.markets import Price from flexmeasures.data.models.weather import WeatherSensor, Weather from flexmeasures.data.utils import save_to_session -from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( unrecognized_sensor, ResponseTuple, @@ -284,7 +283,7 @@ def asset_replace_name_with_id(connections_as_name: List[str]) -> List[str]: def get_weather_sensor_by( weather_sensor_type_name: str, latitude: float = 0, longitude: float = 0 -) -> WeatherSensor: +) -> Union[WeatherSensor, ResponseTuple]: """ Search a weather sensor by type and location. Can create a weather sensor if needed (depends on API mode) @@ -334,29 +333,6 @@ def get_weather_sensor_by( return weather_sensor -def get_generic_asset( - asset_descriptor, entity_type -) -> Union[Asset, Market, WeatherSensor, None]: - """ - Get a generic asset from form information - # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case - """ - ea = parse_entity_address(asset_descriptor, entity_type=entity_type) - if ea is None: - return None - if entity_type == "connection": - return Asset.query.filter(Asset.id == ea["asset_id"]).one_or_none() - elif entity_type == "market": - return Market.query.filter(Market.name == ea["market_name"]).one_or_none() - elif entity_type == "sensor": - return get_weather_sensor_by( - ea["weather_sensor_type_name"], - ea["latitude"], - ea["longitude"], - ) - return None - - def save_to_db( timed_values: List[Union[Power, Price, Weather]], forecasting_jobs: List[Job] ) -> ResponseTuple: diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index 19d5a3323..027e4d079 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -16,6 +16,7 @@ from webargs.flaskparser import parser +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.schemas.times import DurationField from flexmeasures.api.common.responses import ( # noqa: F401 required_info_missing, @@ -40,7 +41,6 @@ parse_as_list, contains_empty_items, upsample_values, - get_generic_asset, ) from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.config import db @@ -736,7 +736,9 @@ def decorated_service(*args, **kwargs): return wrapper -def post_data_checked_for_required_resolution(entity_type): # noqa: C901 +def post_data_checked_for_required_resolution( + entity_type: str, fm_scheme: str +): # noqa: C901 """Decorator which checks that a POST request receives time series data with the event resolutions required by the sensor (asset). It sets the "resolution" keyword argument. If the resolution in the data is a multiple of the asset resolution, values are upsampled to the asset resolution. @@ -794,7 +796,9 @@ def decorated_service(*args, **kwargs): for asset_group in kwargs["generic_asset_name_groups"]: for asset_descriptor in asset_group: # Getting the asset - generic_asset = get_generic_asset(asset_descriptor, entity_type) + generic_asset = SensorField(entity_type, fm_scheme).deserialize( + asset_descriptor + ) if generic_asset is None: return unrecognized_asset( f"Failed to look up asset by {asset_descriptor}" @@ -812,6 +816,7 @@ def decorated_service(*args, **kwargs): last_asset = generic_asset # if inferred resolution is a multiple from required_solution, we can upsample_values + # todo: next line fails on sensors with 0 resolution if inferred_resolution % required_resolution == timedelta(hours=0): for i in range(len(kwargs["value_groups"])): kwargs["value_groups"][i] = upsample_values( @@ -837,7 +842,7 @@ def decorated_service(*args, **kwargs): return wrapper -def get_data_downsampling_allowed(entity_type): +def get_data_downsampling_allowed(entity_type: str, fm_scheme: str): """Decorator which allows downsampling of data which a GET request returns. It checks for a form parameter "resolution". If that is given and is a multiple of the asset's event_resolution, @@ -880,7 +885,9 @@ def decorated_service(*args, **kwargs): # of the event_resolution(s) and thus downsampling is possible) for asset_group in kwargs["generic_asset_name_groups"]: for asset_descriptor in asset_group: - generic_asset = get_generic_asset(asset_descriptor, entity_type) + generic_asset = SensorField(entity_type, fm_scheme).deserialize( + asset_descriptor + ) if generic_asset is None: return unrecognized_asset() asset_resolution = generic_asset.event_resolution diff --git a/flexmeasures/api/v1/implementations.py b/flexmeasures/api/v1/implementations.py index 8ddbc3b7a..9203161b3 100644 --- a/flexmeasures/api/v1/implementations.py +++ b/flexmeasures/api/v1/implementations.py @@ -50,7 +50,7 @@ ) @optional_prior_accepted(ex_post=True, infer_missing=False) @period_required -@get_data_downsampling_allowed("connection") +@get_data_downsampling_allowed("connection", "fm0") @as_json def get_meter_data_response( unit, @@ -101,7 +101,7 @@ def get_meter_data_response( ex_post=True, infer_missing=True, accept_repeating_interval=True ) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm0") @as_json def post_meter_data_response( unit, @@ -184,7 +184,7 @@ def collect_connection_and_value_groups( # Parse the entity address try: connection_details = parse_entity_address( - connection, entity_type="connection" + connection, entity_type="connection", fm_scheme="fm0" ) except EntityAddressException as eae: return invalid_domain(str(eae)) @@ -257,7 +257,9 @@ def create_connection_and_value_groups( # noqa: C901 # TODO: get asset through util function after refactoring # Parse the entity address try: - connection = parse_entity_address(connection, entity_type="connection") + connection = parse_entity_address( + connection, entity_type="connection", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = connection["asset_id"] diff --git a/flexmeasures/api/v1_1/implementations.py b/flexmeasures/api/v1_1/implementations.py index 5c896caba..a99b26811 100644 --- a/flexmeasures/api/v1_1/implementations.py +++ b/flexmeasures/api/v1_1/implementations.py @@ -13,6 +13,7 @@ invalid_domain, invalid_unit, unrecognized_market, + ResponseTuple, invalid_horizon, ) from flexmeasures.api.common.utils.api_utils import ( @@ -66,7 +67,7 @@ def get_connection_response(): @optional_horizon_accepted(infer_missing=True, accept_repeating_interval=True) @values_required @period_required -@post_data_checked_for_required_resolution("market") +@post_data_checked_for_required_resolution("market", "fm0") def post_price_data_response( unit, generic_asset_name_groups, @@ -88,7 +89,7 @@ def post_price_data_response( # Parse the entity address try: - ea = parse_entity_address(market, entity_type="market") + ea = parse_entity_address(market, entity_type="market", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) market_name = ea["market_name"] @@ -141,7 +142,7 @@ def post_price_data_response( @optional_horizon_accepted(infer_missing=True, accept_repeating_interval=True) @values_required @period_required -@post_data_checked_for_required_resolution("sensor") +@post_data_checked_for_required_resolution("weather_sensor", "fm0") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -163,7 +164,9 @@ def post_weather_data_response( # noqa: C901 # Parse the entity address try: - ea = parse_entity_address(sensor, entity_type="sensor") + ea = parse_entity_address( + sensor, entity_type="weather_sensor", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] @@ -178,6 +181,9 @@ def post_weather_data_response( # noqa: C901 weather_sensor = get_weather_sensor_by( weather_sensor_type_name, latitude, longitude ) + if type(weather_sensor) == ResponseTuple: + # Error message telling the user about the nearest weather sensor they can post to + return weather_sensor # Create new Weather objects for j, value in enumerate(value_group): @@ -224,7 +230,7 @@ def post_weather_data_response( # noqa: C901 @optional_horizon_accepted(infer_missing=False, accept_repeating_interval=True) @optional_prior_accepted(infer_missing=False) @period_required -@get_data_downsampling_allowed("connection") +@get_data_downsampling_allowed("connection", "fm0") @as_json def get_prognosis_response( unit, @@ -267,7 +273,7 @@ def get_prognosis_response( ex_post=False, infer_missing=False, accept_repeating_interval=True ) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm0") @as_json def post_prognosis_response( unit, diff --git a/flexmeasures/api/v1_1/tests/conftest.py b/flexmeasures/api/v1_1/tests/conftest.py index b7c5f6c4d..6fa73f7a2 100644 --- a/flexmeasures/api/v1_1/tests/conftest.py +++ b/flexmeasures/api/v1_1/tests/conftest.py @@ -97,8 +97,6 @@ def setup_api_test_data(db, setup_roles_users, add_market_prices): power_forecasts.append(p_3) db.session.bulk_save_objects(power_forecasts) - add_weather_sensors(db) - print("Done setting up data for API v1.1 tests") @@ -107,33 +105,3 @@ def setup_fresh_api_v1_1_test_data( fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db ): return fresh_db - - -def add_weather_sensors(db): - """ Create 2 weather sensors """ - - from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType - - test_sensor_type = WeatherSensorType(name="wind_speed") - db.session.add(test_sensor_type) - sensor = WeatherSensor( - name="wind_speed_sensor", - weather_sensor_type_name="wind_speed", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - unit="m/s", - ) - db.session.add(sensor) - - test_sensor_type = WeatherSensorType(name="temperature") - db.session.add(test_sensor_type) - sensor = WeatherSensor( - name="temperature_sensor", - weather_sensor_type_name="temperature", - event_resolution=timedelta(minutes=5), - latitude=33.4843866, - longitude=126, - unit="°C", - ) - db.session.add(sensor) diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1.py b/flexmeasures/api/v1_1/tests/test_api_v1_1.py index e723ac3f5..62eb083eb 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1.py @@ -3,6 +3,7 @@ from datetime import timedelta from iso8601 import parse_date +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.utils.entity_address_utils import parse_entity_address from flexmeasures.api.common.responses import ( request_processed, @@ -11,7 +12,6 @@ ) from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.common.utils.api_utils import ( - get_generic_asset, message_replace_name_with_ea, ) from flexmeasures.api.v1_1.tests.utils import ( @@ -151,7 +151,7 @@ def test_post_price_data(setup_api_test_data, db, app, clean_redis, post_message ) # only one market is affected, but two horizons horizons = [timedelta(hours=24), timedelta(hours=48)] jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", "fm0").deserialize(post_message["market"]) for job, horizon in zip(jobs, horizons): assert job.kwargs["horizon"] == horizon assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon @@ -177,8 +177,8 @@ def test_post_price_data_invalid_unit(setup_api_test_data, client, post_message) print("Server responded with:\n%s" % post_price_data_response.json) assert post_price_data_response.status_code == 400 assert post_price_data_response.json["type"] == "PostPriceDataResponse" - market = parse_entity_address(post_message["market"], "market") - market_name = market["market_name"] + ea = parse_entity_address(post_message["market"], "market", fm_scheme="fm0") + market_name = ea["market_name"] market = Market.query.filter_by(name=market_name).one_or_none() assert ( post_price_data_response.json["message"] @@ -190,7 +190,9 @@ def test_post_price_data_invalid_unit(setup_api_test_data, client, post_message) "post_message", [message_for_post_weather_data(), message_for_post_weather_data(temperature=True)], ) -def test_post_weather_forecasts(setup_api_test_data, app, client, post_message): +def test_post_weather_forecasts( + setup_api_test_data, add_weather_sensors, app, client, post_message +): """ Try to post wind speed and temperature forecasts as a logged-in test user with the Supplier role, which should succeed. As only forecasts are sent, no forecasting jobs are expected. diff --git a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py index ba5e5bb71..c328a81f5 100644 --- a/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py +++ b/flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py @@ -8,7 +8,6 @@ from flexmeasures.utils.time_utils import forecast_horizons_for from flexmeasures.api.common.responses import unapplicable_resolution from flexmeasures.api.tests.utils import get_auth_token -from flexmeasures.api.v1_1.tests.conftest import add_weather_sensors from flexmeasures.api.v1_1.tests.utils import ( message_for_post_price_data, message_for_post_weather_data, @@ -58,13 +57,16 @@ def test_post_price_data_unexpected_resolution( "post_message", [message_for_post_weather_data(as_forecasts=False)], ) -def test_post_weather_data(setup_fresh_api_v1_1_test_data, app, client, post_message): +def test_post_weather_data( + setup_fresh_api_v1_1_test_data, + add_weather_sensors_fresh_db, + app, + client, + post_message, +): """ Try to post wind speed data as a logged-in test user, which should lead to forecasting jobs. """ - db = setup_fresh_api_v1_1_test_data - add_weather_sensors(db) - auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") post_weather_data_response = client.post( url_for("flexmeasures_api_v1_1.post_weather_data"), diff --git a/flexmeasures/api/v1_1/tests/utils.py b/flexmeasures/api/v1_1/tests/utils.py index aeb6bf776..ab47b7cc9 100644 --- a/flexmeasures/api/v1_1/tests/utils.py +++ b/flexmeasures/api/v1_1/tests/utils.py @@ -1,5 +1,5 @@ """Useful test messages""" -from typing import Optional, Dict, List, Any +from typing import Optional, Dict, Any, List, Union from datetime import timedelta from isodate import duration_isoformat, parse_duration, parse_datetime @@ -8,7 +8,7 @@ from rq.job import Job from flask import current_app -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.models.markets import Market, Price @@ -53,7 +53,7 @@ def message_for_get_prognosis( def message_for_post_price_data( tile_n: int = 1, compress_n: int = 1, - duration: Optional[timedelta] = None, + duration: Optional[Union[timedelta, str]] = None, invalid_unit: bool = False, ) -> dict: """ @@ -62,7 +62,8 @@ def message_for_post_price_data( :param tile_n: Tile the price profile back to back to obtain price data for n days (default = 1). :param compress_n: Compress the price profile to obtain price data with a coarser resolution (default = 1), e.g. compress=4 leads to a resolution of 4 hours. - :param duration: Set a duration explicitly to obtain price data with a coarser or finer resolution + :param duration: timedelta or iso8601 string + Set a duration explicitly to obtain price data with a coarser or finer resolution (the default is equal to 24 hours * tile_n), e.g. (assuming tile_n=1) duration=timedelta(hours=6) leads to a resolution of 15 minutes, and duration=timedelta(hours=48) leads to a resolution of 2 hours. @@ -106,7 +107,11 @@ def message_for_post_price_data( "unit": "EUR/MWh", } if duration is not None: - message["duration"] = duration + message["duration"] = ( + duration_isoformat(duration) + if isinstance(duration, timedelta) + else duration + ) if compress_n > 1: message["values"] = message["values"][::compress_n] if invalid_unit: @@ -148,7 +153,7 @@ def verify_prices_in_db(post_message, values, db, swapped_sign: bool = False): start = parse_datetime(post_message["start"]) end = start + parse_duration(post_message["duration"]) horizon = parse_duration(post_message["horizon"]) - market: Market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", "fm0").deserialize(post_message["market"]) resolution = market.event_resolution query = ( db.session.query(Price.value, Price.horizon) diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py index 6fe1a0c79..2d0697eaa 100644 --- a/flexmeasures/api/v1_2/implementations.py +++ b/flexmeasures/api/v1_2/implementations.py @@ -64,7 +64,7 @@ def get_device_message_response(generic_asset_name_groups, duration): # Parse the entity address try: - ea = parse_entity_address(event, entity_type="event") + ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = ea["asset_id"] @@ -155,7 +155,9 @@ def post_udi_event_response(unit): # noqa: C901 if "event" not in form: return invalid_domain("No event identifier sent.") try: - ea = parse_entity_address(form.get("event"), entity_type="event") + ea = parse_entity_address( + form.get("event"), entity_type="event", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py index a691a5546..42302f9e2 100644 --- a/flexmeasures/api/v1_3/implementations.py +++ b/flexmeasures/api/v1_3/implementations.py @@ -69,7 +69,7 @@ def get_device_message_response(generic_asset_name_groups, duration): # Parse the entity address try: - ea = parse_entity_address(event, entity_type="event") + ea = parse_entity_address(event, entity_type="event", fm_scheme="fm0") except EntityAddressException as eae: return invalid_domain(str(eae)) asset_id = ea["asset_id"] @@ -212,7 +212,9 @@ def post_udi_event_response(unit): if "event" not in form: return invalid_domain("No event identifier sent.") try: - ea = parse_entity_address(form.get("event"), entity_type="event") + ea = parse_entity_address( + form.get("event"), entity_type="event", fm_scheme="fm0" + ) except EntityAddressException as eae: return invalid_domain(str(eae)) diff --git a/flexmeasures/api/v2_0/implementations/sensors.py b/flexmeasures/api/v2_0/implementations/sensors.py index b8a5cb687..25215908e 100644 --- a/flexmeasures/api/v2_0/implementations/sensors.py +++ b/flexmeasures/api/v2_0/implementations/sensors.py @@ -50,7 +50,7 @@ @optional_prior_accepted(infer_missing=True, infer_missing_play=False) @values_required @period_required -@post_data_checked_for_required_resolution("market") +@post_data_checked_for_required_resolution("market", "fm1") def post_price_data_response( # noqa C901 unit, generic_asset_name_groups, @@ -80,12 +80,12 @@ def post_price_data_response( # noqa C901 ea = parse_entity_address(market, entity_type="market") except EntityAddressException as eae: return invalid_domain(str(eae)) - market_name = ea["market_name"] + market_id = ea["sensor_id"] # Look for the Market object - market = Market.query.filter(Market.name == market_name).one_or_none() + market = Market.query.filter(Market.id == market_id).one_or_none() if market is None: - return unrecognized_market(market_name) + return unrecognized_market(market_id) elif unit != market.unit: return invalid_unit("%s prices" % market.display_name, [market.unit]) @@ -129,12 +129,12 @@ def post_price_data_response( # noqa C901 @type_accepted("PostWeatherDataRequest") @unit_required -@assets_required("sensor") +@assets_required("weather_sensor") @optional_horizon_accepted(infer_missing=False, infer_missing_play=True) @optional_prior_accepted(infer_missing=True, infer_missing_play=False) @values_required @period_required -@post_data_checked_for_required_resolution("sensor") +@post_data_checked_for_required_resolution("weather_sensor", "fm1") def post_weather_data_response( # noqa: C901 unit, generic_asset_name_groups, @@ -160,7 +160,7 @@ def post_weather_data_response( # noqa: C901 # Parse the entity address try: - ea = parse_entity_address(sensor, entity_type="sensor") + ea = parse_entity_address(sensor, entity_type="weather_sensor") except EntityAddressException as eae: return invalid_domain(str(eae)) weather_sensor_type_name = ea["weather_sensor_type_name"] @@ -225,7 +225,7 @@ def post_weather_data_response( # noqa: C901 @optional_horizon_accepted(ex_post=True, infer_missing=False, infer_missing_play=True) @optional_prior_accepted(ex_post=True, infer_missing=True, infer_missing_play=False) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm1") @as_json def post_meter_data_response( unit, @@ -257,7 +257,7 @@ def post_meter_data_response( @optional_horizon_accepted(ex_post=False, infer_missing=False, infer_missing_play=False) @optional_prior_accepted(ex_post=False, infer_missing=True, infer_missing_play=False) @period_required -@post_data_checked_for_required_resolution("connection") +@post_data_checked_for_required_resolution("connection", "fm1") @as_json def post_prognosis_response( unit, @@ -314,10 +314,10 @@ def post_power_data( # TODO: get asset through util function after refactoring # Parse the entity address try: - connection = parse_entity_address(connection, entity_type="connection") + ea = parse_entity_address(connection, entity_type="connection") except EntityAddressException as eae: return invalid_domain(str(eae)) - asset_id = connection["asset_id"] + asset_id = ea["sensor_id"] # Look for the Asset object if asset_id in user_asset_ids: diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py index 31dfd55e4..c3a2a70f6 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py @@ -9,15 +9,14 @@ @pytest.mark.parametrize( - "post_message", + "post_message, fm_scheme", [ - message_for_post_prognosis(), + (message_for_post_prognosis(), "fm1"), ], ) -def test_post_prognosis(db, app, post_message): +def test_post_prognosis_2_0(db, app, post_message, fm_scheme): with app.test_client() as client: - # post price data - auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest") + auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest") post_prognosis_response = client.post( url_for("flexmeasures_api_v2_0.post_prognosis"), json=post_message, @@ -32,5 +31,6 @@ def test_post_prognosis(db, app, post_message): post_message["values"], db, entity_type="connection", + fm_scheme=fm_scheme, swapped_sign=True, ) diff --git a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py index 49c2fc50e..87cc7598e 100644 --- a/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py +++ b/flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py @@ -4,7 +4,7 @@ from flask import url_for from iso8601 import parse_date -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.api.tests.utils import get_auth_token from flexmeasures.api.v2_0.tests.utils import ( message_for_post_price_data, @@ -15,8 +15,8 @@ @pytest.mark.parametrize( "post_message", [ - message_for_post_price_data(), - message_for_post_price_data(prior_instead_of_horizon=True), + message_for_post_price_data(market_id=7), + message_for_post_price_data(market_id=1, prior_instead_of_horizon=True), ], ) def test_post_price_data_2_0( @@ -46,7 +46,7 @@ def test_post_price_data_2_0( assert post_price_data_response.json["type"] == "PostPriceDataResponse" verify_sensor_data_in_db( - post_message, post_message["values"], db, entity_type="market" + post_message, post_message["values"], db, entity_type="market", fm_scheme="fm1" ) # look for Forecasting jobs in queue @@ -55,7 +55,7 @@ def test_post_price_data_2_0( ) # only one market is affected, but two horizons horizons = [timedelta(hours=24), timedelta(hours=48)] jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"]) - market = get_generic_asset(post_message["market"], "market") + market = SensorField("market", fm_scheme="fm1").deserialize(post_message["market"]) for job, horizon in zip(jobs, horizons): assert job.kwargs["horizon"] == horizon assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon diff --git a/flexmeasures/api/v2_0/tests/utils.py b/flexmeasures/api/v2_0/tests/utils.py index 410d83f00..215b6d0e8 100644 --- a/flexmeasures/api/v2_0/tests/utils.py +++ b/flexmeasures/api/v2_0/tests/utils.py @@ -1,15 +1,16 @@ -from flexmeasures.data.services.users import find_user_by_email -from typing import Optional +from typing import Optional, Union from datetime import timedelta from isodate import duration_isoformat, parse_duration, parse_datetime import pandas as pd import timely_beliefs as tb -from flexmeasures.api.common.utils.api_utils import get_generic_asset +from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.models.assets import Asset, Power from flexmeasures.data.models.markets import Market, Price +from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.weather import WeatherSensor, Weather +from flexmeasures.data.services.users import find_user_by_email from flexmeasures.api.v1_1.tests.utils import ( message_for_post_price_data as v1_1_message_for_post_price_data, ) @@ -31,6 +32,7 @@ def get_asset_post_data() -> dict: def message_for_post_price_data( + market_id: int, tile_n: int = 1, compress_n: int = 1, duration: Optional[timedelta] = None, @@ -58,6 +60,7 @@ def message_for_post_price_data( duration=duration, invalid_unit=invalid_unit, ) + message["market"] = f"ea1.2018-06.localhost:fm1.{market_id}" message["horizon"] = duration_isoformat(timedelta(hours=0)) if no_horizon or prior_instead_of_horizon: message.pop("horizon", None) @@ -67,16 +70,24 @@ def message_for_post_price_data( def verify_sensor_data_in_db( - post_message, values, db, entity_type: str, swapped_sign: bool = False + post_message, + values, + db, + entity_type: str, + fm_scheme: str, + swapped_sign: bool = False, ): """util method to verify that sensor data ended up in the database""" - if entity_type == "connection": + if entity_type == "sensor": + sensor_type = Sensor + data_type = TimedBelief + elif entity_type == "connection": sensor_type = Asset data_type = Power elif entity_type == "market": sensor_type = Market data_type = Price - elif entity_type == "sensor": + elif entity_type == "weather_sensor": sensor_type = WeatherSensor data_type = Weather else: @@ -84,8 +95,10 @@ def verify_sensor_data_in_db( start = parse_datetime(post_message["start"]) end = start + parse_duration(post_message["duration"]) - market: Market = get_generic_asset(post_message[entity_type], entity_type) - resolution = market.event_resolution + sensor: Union[Sensor, Asset, Market, WeatherSensor] = SensorField( + entity_type, fm_scheme + ).deserialize(post_message[entity_type]) + resolution = sensor.event_resolution if "horizon" in post_message: horizon = parse_duration(post_message["horizon"]) query = ( @@ -95,7 +108,7 @@ def verify_sensor_data_in_db( ) .filter(data_type.horizon == horizon) .join(sensor_type) - .filter(sensor_type.name == market.name) + .filter(sensor_type.name == sensor.name) ) else: query = ( @@ -109,7 +122,7 @@ def verify_sensor_data_in_db( ) # .filter(data_type.horizon == (data_type.datetime + resolution) - prior) # only for sensors with 0-hour ex_post knowledge horizon function .join(sensor_type) - .filter(sensor_type.name == market.name) + .filter(sensor_type.name == sensor.name) ) # todo: after basing Price on TimedBelief, we should be able to get a BeliefsDataFrame from the query directly df = pd.DataFrame( @@ -122,7 +135,7 @@ def verify_sensor_data_in_db( "horizon": "belief_horizon", } ) - bdf = tb.BeliefsDataFrame(df, sensor=market, source="Some source") + bdf = tb.BeliefsDataFrame(df, sensor=sensor, source="Some source") if "prior" in post_message: prior = parse_datetime(post_message["prior"]) bdf = bdf.fixed_viewpoint(prior) @@ -131,11 +144,14 @@ def verify_sensor_data_in_db( assert bdf["event_value"].tolist() == values -def message_for_post_prognosis(): +def message_for_post_prognosis(fm_scheme: str = "fm1"): + """ + Posting prognosis for a wind turbine's production. + """ message = { "type": "PostPrognosisRequest", - "connection": "ea1.2018-06.localhost:2:5", - "values": [300, 300, 300, 0, 0, 300], + "connection": f"ea1.2018-06.localhost:{fm_scheme}.2", + "values": [-300, -300, -300, 0, 0, -300], "start": "2021-01-01T00:00:00Z", "duration": "PT1H30M", "prior": "2020-12-31T18:00:00Z", diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 15c1dd3d6..c937cc17c 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -24,6 +24,7 @@ from flexmeasures.data.services.users import create_user from flexmeasures.data.models.assets import AssetType, Asset, Power from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType from flexmeasures.data.models.markets import Market, MarketType, Price from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.user import User @@ -448,6 +449,56 @@ def add_charging_station_assets( } +@pytest.fixture(scope="module") +def add_weather_sensors(db) -> Dict[str, WeatherSensor]: + return create_weather_sensors(db) + + +@pytest.fixture(scope="function") +def add_weather_sensors_fresh_db(fresh_db) -> Dict[str, WeatherSensor]: + return create_weather_sensors(fresh_db) + + +def create_weather_sensors(db: SQLAlchemy): + """Add some weather sensors and weather sensor types.""" + + test_sensor_type = WeatherSensorType(name="wind_speed") + db.session.add(test_sensor_type) + wind_sensor = WeatherSensor( + name="wind_speed_sensor", + weather_sensor_type_name="wind_speed", + event_resolution=timedelta(minutes=5), + latitude=33.4843866, + longitude=126, + unit="m/s", + ) + db.session.add(wind_sensor) + + test_sensor_type = WeatherSensorType(name="temperature") + db.session.add(test_sensor_type) + temp_sensor = WeatherSensor( + name="temperature_sensor", + weather_sensor_type_name="temperature", + event_resolution=timedelta(minutes=5), + latitude=33.4843866, + longitude=126.0, + unit="°C", + ) + db.session.add(temp_sensor) + return {"wind": wind_sensor, "temperature": temp_sensor} + + +@pytest.fixture(scope="module") +def add_sensors(db: SQLAlchemy): + """Add some generic sensors.""" + height_sensor = Sensor( + name="my daughter's height", + unit="m", + ) + db.session.add(height_sensor) + return height_sensor + + @pytest.fixture(scope="function") def clean_redis(app): failed = app.queues["forecasting"].failed_job_registry diff --git a/flexmeasures/data/models/assets.py b/flexmeasures/data/models/assets.py index c547bd427..7dd30712e 100644 --- a/flexmeasures/data/models/assets.py +++ b/flexmeasures/data/models/assets.py @@ -134,11 +134,19 @@ def power_unit(self) -> float: return self.unit @property - def entity_address(self) -> str: + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" return build_entity_address( - dict(owner_id=self.owner_id, asset_id=self.id), "connection" + dict(owner_id=self.owner_id, asset_id=self.id), + "connection", + fm_scheme="fm0", ) + @property + def entity_address(self) -> str: + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address(dict(sensor_id=self.id), "sensor") + @property def location(self) -> Tuple[float, float]: return self.latitude, self.longitude diff --git a/flexmeasures/data/models/markets.py b/flexmeasures/data/models/markets.py index 181a04efc..dd32431c2 100644 --- a/flexmeasures/data/models/markets.py +++ b/flexmeasures/data/models/markets.py @@ -6,6 +6,7 @@ from flexmeasures.data.config import db from flexmeasures.data.models.time_series import Sensor, TimedValue +from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -78,6 +79,18 @@ def __init__(self, **kwargs): if "display_name" not in kwargs: self.display_name = humanize(self.name) + @property + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" + return build_entity_address( + dict(market_name=self.name), "market", fm_scheme="fm0" + ) + + @property + def entity_address(self) -> str: + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address(dict(sensor_id=self.id), "sensor") + @property def price_unit(self) -> str: """Return the 'unit' property of the generic asset, just with a more insightful name.""" diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 88c3290df..4e5284270 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -16,6 +16,7 @@ exclude_source_type_filter, ) from flexmeasures.data.services.time_series import collect_time_series_data +from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.data.models.charts import chart_type_to_chart_specs from flexmeasures.utils.time_utils import server_now from flexmeasures.utils.flexmeasures_inflection import capitalize @@ -29,6 +30,10 @@ def __init__(self, name: str, **kwargs): tb_utils.remove_class_init_kwargs(tb.SensorDBMixin, kwargs) db.Model.__init__(self, **kwargs) + @property + def entity_address(self) -> str: + return build_entity_address(dict(sensor_id=self.id), "sensor") + def search_beliefs( self, event_starts_after: Optional[datetime_type] = None, @@ -116,7 +121,7 @@ def chart( @property def timerange(self) -> Dict[str, datetime_type]: - """Timerange for which sensor data exists. + """Time range for which sensor data exists. :returns: dictionary with start and end, for example: { diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index b1d4505a4..b844a5818 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -8,9 +8,9 @@ from sqlalchemy.schema import UniqueConstraint from flexmeasures.data.config import db - from flexmeasures.data.models.time_series import Sensor, TimedValue from flexmeasures.utils.geo_utils import parse_lat_lng +from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -79,6 +79,27 @@ def __init__(self, **kwargs): self.id = new_sensor_id self.name = self.name.replace(" ", "_").lower() + @property + def entity_address_fm0(self) -> str: + """Entity address under the fm0 scheme for entity addresses.""" + return build_entity_address( + dict( + weather_sensor_type_name=self.weather_sensor_type_name, + latitude=self.latitude, + longitude=self.longitude, + ), + "weather_sensor", + fm_scheme="fm0", + ) + + @property + def entity_address(self) -> str: + """Entity address under the latest fm scheme for entity addresses.""" + return build_entity_address( + dict(sensor_id=self.id), + "sensor", + ) + @property def weather_unit(self) -> float: """Return the 'unit' property of the generic asset, just with a more insightful name.""" @@ -114,7 +135,7 @@ def great_circle_distance(self, **kwargs): great_circle_distance(lat=32, lng=54) """ - r = 6371 # Radius of Earth in kilometers + r = 6371 # Radius of Earth in kilometres other_latitude, other_longitude = parse_lat_lng(kwargs) if other_latitude is None or other_longitude is None: return None diff --git a/flexmeasures/data/schemas/tests/test_times.py b/flexmeasures/data/schemas/tests/test_times.py index a834effc8..16e5dcd68 100644 --- a/flexmeasures/data/schemas/tests/test_times.py +++ b/flexmeasures/data/schemas/tests/test_times.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize( - "duration_input,exp_deserialization", + "duration_input, exp_deserialization", [ ("PT1H", timedelta(hours=1)), ("PT6M", timedelta(minutes=6)), @@ -25,7 +25,7 @@ def test_duration_field_straightforward(duration_input, exp_deserialization): @pytest.mark.parametrize( - "duration_input,exp_deserialization,grounded_timedelta", + "duration_input, exp_deserialization, grounded_timedelta", [ ("P1M", isodate.Duration(months=1), timedelta(days=29)), ("PT24H", isodate.Duration(hours=24), timedelta(hours=24)), @@ -59,7 +59,7 @@ def test_duration_field_nominal_grounded( @pytest.mark.parametrize( - "duration_input,error_msg", + "duration_input, error_msg", [ ("", "Unable to parse duration string"), ("1H", "Unable to parse duration string"), diff --git a/flexmeasures/data/scripts/cli_tasks/data_add.py b/flexmeasures/data/scripts/cli_tasks/data_add.py index 729d95103..27dc69851 100644 --- a/flexmeasures/data/scripts/cli_tasks/data_add.py +++ b/flexmeasures/data/scripts/cli_tasks/data_add.py @@ -213,9 +213,8 @@ def add_weather_sensor(**args): sensor = WeatherSensor(**args) app.db.session.add(sensor) app.db.session.commit() - print(f"Successfully created sensor with ID {sensor.id}") - # TODO: uncomment when #66 has landed - # print(f"You can access it at its entity address {sensor.entity_address}") + print(f"Successfully created weather sensor with ID {sensor.id}") + print(f" You can access it at its entity address {sensor.entity_address}") @fm_add_data.command("structure") diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 900eef8ae..969d34bc8 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -110,7 +110,9 @@ def add_test_weather_sensor_and_forecasts(db: SQLAlchemy): name="Seita", type="demo script" ).one_or_none() for sensor_name in ("radiation", "wind_speed"): - sensor_type = WeatherSensorType(name=sensor_name) + sensor_type = WeatherSensorType.query.filter_by(name=sensor_name).one_or_none() + if sensor_type is None: + sensor_type = WeatherSensorType(name=sensor_name) sensor = WeatherSensor( name=sensor_name, sensor_type=sensor_type, latitude=100, longitude=100 ) diff --git a/flexmeasures/ui/templates/crud/assets.html b/flexmeasures/ui/templates/crud/assets.html index 7324c5c31..a44e932c6 100644 --- a/flexmeasures/ui/templates/crud/assets.html +++ b/flexmeasures/ui/templates/crud/assets.html @@ -21,6 +21,7 @@

All assets

Asset id Owner id Entity address + Old entity address (API v1) {% if user_is_admin %}
@@ -57,6 +58,9 @@

All assets

{{ asset.entity_address }} + + {{ asset.entity_address_fm0 }} + diff --git a/flexmeasures/utils/entity_address_utils.py b/flexmeasures/utils/entity_address_utils.py index 6231e98fa..3f28abed4 100644 --- a/flexmeasures/utils/entity_address_utils.py +++ b/flexmeasures/utils/entity_address_utils.py @@ -11,60 +11,55 @@ """ -Functionality to support parsing and building USEF's EA1 addressing scheme [1], -which is mostly taken from IETF RFC 3720 [2]: - -This is the complete structure of an EA1 address: - -ea1.{date code}.{reversed domain name}:{locally unique string} - -for example "ea1.2021-01.io.flexmeasures.company:sensor14" - -- "ea1" is a constant, indicating this is a type 1 USEF entity address -- The date code "must be a date during which the naming authority owned - the domain name used in this format, and should be the first month in which the domain name was - owned by this naming authority at 00:01 GMT of the first day of the month. -- The reversed domain name is taken from the naming authority - (person or organization) creating this entity address -- The locally unique string can be used for local purposes, and FlexMeasures - uses it to identify the resource (more information in parse_entity_address). - Fields in the locally unique string are separated by colons, see for other examples - IETF RFC 3721, page 6 [3]. - ([2] says it's possible to use dashes, dots or colons ― dashes and dots might come up in - latitude/longitude coordinates of sensors) - -TODO: This needs to be in the FlexMeasures documentation. +Functionality to support parsing and building Entity Addresses as defined by USEF [1]. +See our documentation for more details. [1] https://www.usef.energy/app/uploads/2020/01/USEF-Flex-Trading-Protocol-Specifications-1.01.pdf -[2] https://tools.ietf.org/html/rfc3720 -[3] https://tools.ietf.org/html/rfc3721 """ ADDR_SCHEME = "ea1" +FM1_ADDR_SCHEME = "fm1" +FM0_ADDR_SCHEME = "fm0" class EntityAddressException(Exception): pass +def get_host() -> str: + """Get host from the context of the request. + + Strips off www. but keeps subdomains. + Can be localhost, too. + """ + if has_request_context(): + host = urlparse(request.url).netloc.lstrip("www.") + if host[:9] != "127.0.0.1": + return host + # Assume localhost (for CLI/tests/simulations) + return "localhost" + + def build_entity_address( - entity_info: dict, entity_type: str, host: Optional[str] = None + entity_info: dict, + entity_type: str, + host: Optional[str] = None, + fm_scheme: str = FM1_ADDR_SCHEME, ) -> str: """ Build an entity address. + fm1 type entity address should use entity_info["sensor_id"] + todo: implement entity addresses for actuators with entity_info["actuator_id"] (first ensuring globally unique ids across sensors and actuators) + If the host is not given, it is attempted to be taken from the request. entity_info is expected to contain the required fields for the custom string. Returns the address as string. """ if host is None: - if has_request_context(): - host = urlparse(request.url).netloc - else: - # Assume localhost (for CLI/tests/simulations) - host = "localhost" + host = get_host() def build_field(field: str, required: bool = True): if required and field not in entity_info: @@ -73,13 +68,21 @@ def build_field(field: str, required: bool = True): ) if field not in entity_info: return "" - return f":{entity_info[field]}" + return f"{entity_info[field]}:" - if entity_type == "connection": + if fm_scheme == FM1_ADDR_SCHEME: # and entity_type == "sensor": + locally_unique_str = f"{build_field('sensor_id')}" + # elif fm_scheme == FM1_ADDR_SCHEME and entity_type == "actuator": + # locally_unique_str = f"{build_field('actuator_id')}" + elif fm_scheme != FM0_ADDR_SCHEME: + raise EntityAddressException( + f"Unrecognized FlexMeasures scheme for entity addresses: {fm_scheme}" + ) + elif entity_type == "connection": locally_unique_str = ( f"{build_field('owner_id', required=False)}{build_field('asset_id')}" ) - elif entity_type == "sensor": + elif entity_type == "weather_sensor": locally_unique_str = f"{build_field('weather_sensor_type_name')}{build_field('latitude')}{build_field('longitude')}" elif entity_type == "market": locally_unique_str = f"{build_field('market_name')}" @@ -87,74 +90,139 @@ def build_field(field: str, required: bool = True): locally_unique_str = f"{build_field('owner_id', required=False)}{build_field('asset_id')}{build_field('event_id')}{build_field('event_type')}" else: raise EntityAddressException(f"Unrecognized entity type: {entity_type}") - return build_ea_scheme_and_naming_authority(host) + locally_unique_str - - -def parse_entity_address(entity_address: str, entity_type: str) -> dict: + return ( + build_ea_scheme_and_naming_authority(host) + + ":" + + fm_scheme + + "." + + locally_unique_str.rstrip(":") + ) + + +def parse_entity_address( # noqa: C901 + entity_address: str, + entity_type: str, + fm_scheme: str = FM1_ADDR_SCHEME, +) -> dict: """ Parses a generic asset name into an info dict. - The entity_address must be a valid type 1 USEF entity address. - That is, it must follow the EA1 addressing scheme recommended by USEF. - In addition, FlexMeasures expects the identifying string to contain information in - a certain structure. + Returns a dictionary with scheme, naming_authority and various other fields, + depending on the entity type and FlexMeasures scheme (see examples above). + Returns None if entity type is unknown or entity_address is not parse-able. + We recommend to `return invalid_domain()` in that case. - For example: + Examples for the fm1 scheme: - connection = ea1.2018-06.localhost:40:30 - connection = ea1.2018-06.io.flexmeasures:: - sensor = ea1.2018-06.io.flexmeasures:temperature:52:73.0 - sensor = ea1.2018-06.io.flexmeasures::: - market = ea1.2018-06.io.flexmeasures:epex_da - market = ea1.2018-06.io.flexmeasures: - event = ea1.2018-06.io.flexmeasures:40:30:302:soc - event = ea1.2018-06.io.flexmeasures:::: + sensor = ea1.2021-01.io.flexmeasures:fm1.42 + sensor = ea1.2021-01.io.flexmeasures:fm1. + connection = ea1.2021-01.io.flexmeasures:fm1. + market = ea1.2021-01.io.flexmeasures:fm1. + weather_station = ea1.2021-01.io.flexmeasures:fm1. + todo: UDI events are not yet modelled in the fm1 scheme, but will probably be ea1.2021-01.io.flexmeasures:fm1. - Returns a dictionary with scheme, naming_authority and various other fields, - depending on the entity type (see examples above). - Returns None if entity type is unknown or entity_address is not parseable. - We recommend to `return invalid_domain()` in that case. + Examples for the fm0 scheme: + + connection = ea1.2021-01.localhost:fm0.40:30 + connection = ea1.2021-01.io.flexmeasures:fm0.: + weather_sensor = ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0 + weather_sensor = ea1.2021-01.io.flexmeasures:fm0.:: + market = ea1.2021-01.io.flexmeasures:fm0.epex_da + market = ea1.2021-01.io.flexmeasures:fm0. + event = ea1.2021-01.io.flexmeasures:fm0.40:30:302:soc + event = ea1.2021-01.io.flexmeasures:fm0.::: + + For the fm0 scheme, the 'fm0.' part is optional, for backwards compatibility. """ - # we can rigidly test the start + + # Check the scheme and naming authority date if not entity_address.startswith(ADDR_SCHEME): raise EntityAddressException( f"A valid type 1 USEF entity address starts with '{ADDR_SCHEME}', please review {entity_address}" ) - date_regex = r"[0-9]{4}-[0-9]{2}" + date_regex = r"([0-9]{4})-(0[1-9]|1[012])" if not re.search(fr"^{date_regex}$", entity_address[4:11]): raise EntityAddressException( - f"After '{ADDR_SCHEME}.', a date spec of the format {date_regex} is expected." + f"After '{ADDR_SCHEME}.', a date specification of the format {date_regex} is expected." ) - # Also the entity type - if entity_type not in ("connection", "sensor", "market", "event"): + + # Check the entity type + if entity_type not in ("sensor", "connection", "weather_sensor", "market", "event"): raise EntityAddressException(f"Unrecognized entity type: {entity_type}") - if entity_type == "connection": + def validate_ea_for_fm_scheme(ea: dict, fm_scheme: str): + if "fm_scheme" not in ea: + # Backwards compatibility: assume fm0 if fm_scheme is not specified + ea["fm_scheme"] = FM0_ADDR_SCHEME + scheme = ea["scheme"] + naming_authority = ea["naming_authority"] + if ea["fm_scheme"] != fm_scheme: + raise EntityAddressException( + f"A valid type {fm_scheme[2:]} FlexMeasures entity address starts with '{scheme}.{naming_authority}:{fm_scheme}', please review {entity_address}" + ) + + if fm_scheme == FM1_ADDR_SCHEME: + + # Check the FlexMeasures scheme + if entity_address.split(":")[1][: len(fm_scheme) + 1] != FM1_ADDR_SCHEME + ".": + raise EntityAddressException( + f"A valid type {fm_scheme[2:]} FlexMeasures entity address starts with '{build_ea_scheme_and_naming_authority(get_host())}:{fm_scheme}.', please review {entity_address}" + ) + + match = re.search( + r"^" + r"(?P.+)\." + fr"(?P{date_regex}\.[^:]+)" # everything until the colon (no port) + r":" + r"((?P.+)\.)" + r"(?P\d+)" + r"$", + entity_address, + ) + if match is None: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + value_types = { + "scheme": str, + "naming_authority": str, + "fm_scheme": str, + "sensor_id": int, + } + elif fm_scheme != FM0_ADDR_SCHEME: + raise EntityAddressException( + f"Unrecognized FlexMeasures scheme for entity addresses: {fm_scheme}" + ) + elif entity_type == "connection": match = re.search( r"^" r"(?P.+)\." fr"(?P{date_regex}\.[^:]+)" # everything until the colon (no port) r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"((?P\d+):)*" # owner id is optional r"(?P\d+)" r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "owner_id": int, - "asset_id": int, - } - return _typed_regex_results(match, value_types) - elif entity_type == "sensor": + if match is None: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + value_types = { + "scheme": str, + "naming_authority": str, + "owner_id": int, + "asset_id": int, + } + elif entity_type == "weather_sensor": match = re.search( r"^" r"(?P.+)" r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"(?=[a-zA-Z])(?P[\w]+)" # should start with at least one letter r":" r"(?P\-?\d+(\.\d+)?)" @@ -163,15 +231,17 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "weather_sensor_type_name": str, - "latitude": float, - "longitude": float, - } - return _typed_regex_results(match, value_types) + if match is None: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + value_types = { + "scheme": str, + "naming_authority": str, + "weather_sensor_type_name": str, + "latitude": float, + "longitude": float, + } elif entity_type == "market": match = re.search( r"^" @@ -179,13 +249,16 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"(?=[a-zA-Z])(?P[\w]+)" # should start with at least one letter r"$", entity_address, ) - if match: - value_types = {"scheme": str, "naming_authority": str, "market_name": str} - return _typed_regex_results(match, value_types) + if match is None: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + value_types = {"scheme": str, "naming_authority": str, "market_name": str} elif entity_type == "event": match = re.search( r"^" @@ -193,6 +266,7 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: r"\." fr"(?P{date_regex}\.[^:]+)" r":" + r"((?P.+)\.)*" # for backwards compatibility, missing fm_scheme is interpreted as fm0 r"((?P\d+):)*" # owner id is optional r"(?P\d+)" r":" @@ -202,19 +276,25 @@ def parse_entity_address(entity_address: str, entity_type: str) -> dict: r"$", entity_address, ) - if match: - value_types = { - "scheme": str, - "naming_authority": str, - "owner_id": int, - "asset_id": int, - "event_id": int, - "event_type": str, - } - return _typed_regex_results(match, value_types) + if match is None: + raise EntityAddressException( + f"Could not parse {entity_type} {entity_address}." + ) + value_types = { + "scheme": str, + "naming_authority": str, + "owner_id": int, + "asset_id": int, + "event_id": int, + "event_type": str, + } + else: + # Finally, we simply raise without precise information what went wrong + raise EntityAddressException(f"Could not parse {entity_address}.") - # Finally, we simply raise without precise information what went wrong - raise EntityAddressException(f"Could not parse {entity_address}.") + ea = _typed_regex_results(match, value_types) + validate_ea_for_fm_scheme(ea, fm_scheme) + return ea def build_ea_scheme_and_naming_authority( @@ -236,14 +316,14 @@ def build_ea_scheme_and_naming_authority( [domain_parts.subdomain, domain_parts.domain, domain_parts.suffix], ) ) - if config_var_domain_key in current_app.config.get( + if domain_parts.domain in ("localhost", "127.0.0.1"): + host_auth_start_month = get_first_day_of_next_month().strftime("%Y-%m") + elif config_var_domain_key in current_app.config.get( "FLEXMEASURES_HOSTS_AND_AUTH_START", {} ): host_auth_start_month = current_app.config.get( "FLEXMEASURES_HOSTS_AND_AUTH_START", {} )[config_var_domain_key] - elif domain_parts.domain in ("localhost", "127.0.0.1"): - host_auth_start_month = get_first_day_of_next_month().strftime("%Y-%m") else: raise Exception( f"Could not find out when authority for {config_var_domain_key} started. Is FLEXMEASURES_HOSTS_AND_AUTH_START configured for it?" diff --git a/flexmeasures/utils/tests/test_entity_address_utils.py b/flexmeasures/utils/tests/test_entity_address_utils.py index 0e83826db..b13a249bf 100644 --- a/flexmeasures/utils/tests/test_entity_address_utils.py +++ b/flexmeasures/utils/tests/test_entity_address_utils.py @@ -11,25 +11,56 @@ @pytest.mark.parametrize( - "info, entity_type, host, exp_result", + "info, entity_type, host, fm_scheme, exp_result", [ + ( + dict(sensor_id=42), + "sensor", + "flexmeasures.io", + "fm1", + "ea1.2021-01.io.flexmeasures:fm1.42", + ), + ( + dict(asset_id=42), + "sensor", + "flexmeasures.io", + "fm1", + "required field 'sensor_id'", + ), + ( + dict(sensor_id=40), + "connection", + "flexmeasures.io", + "fm1", + "ea1.2021-01.io.flexmeasures:fm1.40", + ), + ( + dict(asset_id=40), + "connection", + "flexmeasures.io", + "fm1", + "required field 'sensor_id'", + ), ( dict(owner_id=3, asset_id=40), "connection", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:3:40", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.3:40", ), ( dict(owner_id=3), "connection", "flexmeasures.io", + "fm0", "required field 'asset_id'", ), ( dict(owner_id=40, asset_id=30), "connection", "localhost:5000", - f"ea1.{get_first_day_of_next_month().strftime('%Y-%m')}.localhost:40:30", + "fm0", + f"ea1.{get_first_day_of_next_month().strftime('%Y-%m')}.localhost:fm0.40:30", ), ( dict( @@ -37,15 +68,17 @@ latitude=52, longitude=73.0, ), - "sensor", + "weather_sensor", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:temperature:52:73.0", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.temperature:52:73.0", ), ( dict(market_name="epex_da"), "market", "flexmeasures.io", - "ea1.2021-01.io.flexmeasures:epex_da", + "fm0", + "ea1.2021-01.io.flexmeasures:fm0.epex_da", ), ( dict( @@ -56,12 +89,13 @@ ), "event", "http://staging.flexmeasures.io:4444", - "ea1.2022-09.io.flexmeasures.staging:40:30:302:soc", + "fm0", + "ea1.2022-09.io.flexmeasures.staging:fm0.40:30:302:soc", ), ], ) def test_build_entity_address( - app, info: dict, entity_type: str, host: str, exp_result: str + app, info: dict, entity_type: str, host: str, fm_scheme: str, exp_result: str ): with app.app_context(): app.config["FLEXMEASURES_HOSTS_AND_AUTH_START"] = { @@ -69,10 +103,12 @@ def test_build_entity_address( "staging.flexmeasures.io": "2022-09", } if exp_result.startswith("ea1"): - assert build_entity_address(info, entity_type, host) == exp_result + assert ( + build_entity_address(info, entity_type, host, fm_scheme) == exp_result + ) else: with pytest.raises(EntityAddressException, match=exp_result): - build_entity_address(info, entity_type, host) == exp_result + build_entity_address(info, entity_type, host, fm_scheme) == exp_result @pytest.mark.parametrize( @@ -86,10 +122,10 @@ def test_build_entity_address( ( "connection", "ea1.2018-RR.localhost:40:30", - "date spec of the format", + "date specification", ), ( - "sensor", + "weather_sensor", "ea1.2018-04.localhost:5000:40:30", "Could not parse", # no sensor type (which starts with a letter) ), @@ -110,7 +146,7 @@ def test_build_entity_address( dict(naming_authority="2018-06.io.flexmeasures", owner_id=40, asset_id=30), ), ( - "sensor", + "weather_sensor", "ea1.2018-06.io.flexmeasures:temperature:-52:73.0", dict( naming_authority="2018-06.io.flexmeasures", @@ -146,14 +182,28 @@ def test_build_entity_address( event_id=302, ), ), + ( + "connection", + "ea1.2018-06.io.flexmeasures.staging:fm1.30", + dict( + naming_authority="2018-06.io.flexmeasures.staging", + asset_id=30, + owner_id=None, + event_type="connection", + ), + ), ], ) def test_parse_entity_address(entity_type, entity_address, exp_result): if isinstance(exp_result, str): # this means we expect an exception with pytest.raises(EntityAddressException, match=exp_result): - parse_entity_address(entity_address, entity_type=entity_type) + parse_entity_address( + entity_address, entity_type=entity_type, fm_scheme="fm0" + ) else: - res = parse_entity_address(entity_address, entity_type=entity_type) + res = parse_entity_address( + entity_address, entity_type=entity_type, fm_scheme="fm0" + ) assert res["scheme"] == "ea1" assert res["naming_authority"] == exp_result["naming_authority"] if entity_type in ("connection", "event"): @@ -161,7 +211,7 @@ def test_parse_entity_address(entity_type, entity_address, exp_result): assert res[field] == exp_result[field] if entity_type == "market": assert res["market_name"] == exp_result["market_name"] - if entity_type == "sensor": + if entity_type == "weather_sensor": for field in ("weather_sensor_type_name", "latitude", "longitude"): assert res[field] == exp_result[field] if entity_type == "event":