Issue 6 entity address scheme improvements 2

flexmeasures/api/common/schemas/sensors.py

@@ -19,7 +19,7 @@ class EntityAddressValidationError(FMValidationError):
 
 
 class SensorField(fields.Str):
-    """Field that deserializes to a Sensor, Asset, Market or WeatherSensor
+    """Field that de-serializes to a Sensor, Asset, Market or WeatherSensor
     and serializes back to an entity address (string)."""
 
     # todo: when Actuators also get an entity address, refactor this class to EntityField,
@@ -43,7 +43,7 @@ def __init__(
     def _deserialize(  # noqa: C901 todo: the noqa can probably be removed after refactoring Asset/Market/WeatherSensor to Sensor
         self, value, attr, obj, **kwargs
     ) -> Union[Sensor, Asset, Market, WeatherSensor]:
-        """Deserialize to a Sensor, Asset, Market or WeatherSensor."""
+        """De-serialize to a Sensor, Asset, Market or WeatherSensor."""
         # TODO: After refactoring, unify 3 generic_asset cases -> 1 sensor case
         try:
             ea = parse_entity_address(value, self.entity_type, self.fm_scheme)

flexmeasures/api/common/schemas/tests/test_sensors.py

@@ -11,7 +11,7 @@
     "entity_address, entity_type, fm_scheme, exp_deserialization_name",
     [
         (
-            build_entity_address(dict(sensor_id=9), "sensor"),
+            build_entity_address(dict(sensor_id=1), "sensor"),
             "sensor",
             "fm1",
             "my daughter's height",
@@ -26,7 +26,7 @@
         ),
         (
             build_entity_address(
-                dict(owner_id=1, asset_id=3), "connection", fm_scheme="fm0"
+                dict(owner_id=1, asset_id=4), "connection", fm_scheme="fm0"
             ),
             "connection",
             "fm0",
@@ -49,7 +49,14 @@
     ],
 )
 def test_sensor_field_straightforward(
-    entity_address, entity_type, fm_scheme, exp_deserialization_name
+    add_sensors,
+    setup_markets,
+    add_battery_assets,
+    add_weather_sensors,
+    entity_address,
+    entity_type,
+    fm_scheme,
+    exp_deserialization_name,
 ):
     """Testing straightforward cases"""
     sf = SensorField(entity_type, fm_scheme)

flexmeasures/api/common/utils/api_utils.py

@@ -13,7 +13,6 @@
 from flexmeasures.data import db
 from flexmeasures.data.models.assets import Asset, Power
 from flexmeasures.data.models.markets import Price
-from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.weather import WeatherSensor, Weather
 from flexmeasures.data.utils import save_to_session
 from flexmeasures.api.common.responses import (

flexmeasures/api/common/utils/validators.py

@@ -17,7 +17,7 @@
 from webargs.flaskparser import parser
 
 from flexmeasures.api.common.schemas.sensors import SensorField
-from flexmeasures.api.common.schemas.times import DurationField
+from flexmeasures.data.schemas.times import DurationField
 from flexmeasures.api.common.responses import (  # noqa: F401
     required_info_missing,
     invalid_horizon,

flexmeasures/api/v1_1/implementations.py

@@ -14,6 +14,7 @@
     invalid_unit,
     unrecognized_market,
     ResponseTuple,
+    invalid_horizon,
 )
 from flexmeasures.api.common.utils.api_utils import (
     save_to_db,

flexmeasures/api/v1_1/tests/conftest.py

@@ -98,3 +98,10 @@ def setup_api_test_data(db, setup_roles_users, add_market_prices):
     db.session.bulk_save_objects(power_forecasts)
 
     print("Done setting up data for API v1.1 tests")
+
+
+@pytest.fixture(scope="function")
+def setup_fresh_api_v1_1_test_data(
+    fresh_db, setup_roles_users_fresh_db, setup_markets_fresh_db
+):
+    return fresh_db

flexmeasures/api/v1_1/tests/test_api_v1_1.py

@@ -190,7 +190,9 @@ def test_post_price_data_invalid_unit(setup_api_test_data, client, post_message)
     "post_message",
     [message_for_post_weather_data(), message_for_post_weather_data(temperature=True)],
 )
-def test_post_weather_forecasts(setup_api_test_data, app, client, post_message):
+def test_post_weather_forecasts(
+    setup_api_test_data, add_weather_sensors, app, client, post_message
+):
     """
     Try to post wind speed and temperature forecasts as a logged-in test user with the Supplier role, which should succeed.
     As only forecasts are sent, no forecasting jobs are expected.

flexmeasures/api/v1_1/tests/test_api_v1_1_fresh_db.py

@@ -8,7 +8,6 @@
 from flexmeasures.utils.time_utils import forecast_horizons_for
 from flexmeasures.api.common.responses import unapplicable_resolution
 from flexmeasures.api.tests.utils import get_auth_token
-from flexmeasures.api.v1_1.tests.conftest import add_weather_sensors
 from flexmeasures.api.v1_1.tests.utils import (
     message_for_post_price_data,
     message_for_post_weather_data,
@@ -58,13 +57,16 @@ def test_post_price_data_unexpected_resolution(
     "post_message",
     [message_for_post_weather_data(as_forecasts=False)],
 )
-def test_post_weather_data(setup_fresh_api_v1_1_test_data, app, client, post_message):
+def test_post_weather_data(
+    setup_fresh_api_v1_1_test_data,
+    add_weather_sensors_fresh_db,
+    app,
+    client,
+    post_message,
+):
     """
     Try to post wind speed data as a logged-in test user, which should lead to forecasting jobs.
     """
-    db = setup_fresh_api_v1_1_test_data
-    add_weather_sensors(db)
-
     auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest")
     post_weather_data_response = client.post(
         url_for("flexmeasures_api_v1_1.post_weather_data"),

flexmeasures/api/v1_1/tests/utils.py

@@ -1,5 +1,5 @@
 """Useful test messages"""
-from typing import Optional, Dict, Any, Union
+from typing import Optional, Dict, Any, List, Union
 from datetime import timedelta
 from isodate import duration_isoformat, parse_duration, parse_datetime
 

flexmeasures/api/v2_0/implementations/sensors.py

@@ -130,8 +130,8 @@ def post_price_data_response(  # noqa C901
 @type_accepted("PostWeatherDataRequest")
 @unit_required
 @assets_required("weather_sensor")
-@optional_horizon_accepted()
-@optional_prior_accepted()
+@optional_horizon_accepted(infer_missing=False, infer_missing_play=True)
+@optional_prior_accepted(infer_missing=True, infer_missing_play=False)
 @values_required
 @period_required
 @post_data_checked_for_required_resolution("weather_sensor", "fm1")

flexmeasures/api/v2_0/tests/test_api_v2_0_sensors.py

@@ -1,57 +1,13 @@
 from flask import url_for
 import pytest
 
-from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.tests.utils import get_auth_token
 from flexmeasures.api.v2_0.tests.utils import (
     message_for_post_prognosis,
     verify_sensor_data_in_db,
 )
 
 
-@pytest.mark.parametrize(
-    "post_message",
-    [
-        message_for_post_price_data(),
-        message_for_post_price_data(prior_instead_of_horizon=True),
-    ],
-)
-def test_post_price_data_2_0(db, app, post_message):
-    """
-    Try to post price data as a logged-in test user with the Supplier role, which should succeed.
-    """
-    # call with client whose context ends, so that we can test for,
-    # after-effects in the database after teardown committed.
-    with app.test_client() as client:
-        # post price data
-        auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest")
-        post_price_data_response = client.post(
-            url_for("flexmeasures_api_v2_0.post_price_data"),
-            json=post_message,
-            headers={"Authorization": auth_token},
-        )
-        print("Server responded with:\n%s" % post_price_data_response.json)
-        assert post_price_data_response.status_code == 200
-        assert post_price_data_response.json["type"] == "PostPriceDataResponse"
-
-    verify_sensor_data_in_db(
-        post_message, post_message["values"], db, entity_type="market", fm_scheme="fm1"
-    )
-
-    # look for Forecasting jobs in queue
-    assert (
-        len(app.queues["forecasting"]) == 2
-    )  # only one market is affected, but two horizons
-    horizons = [timedelta(hours=24), timedelta(hours=48)]
-    jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"])
-    market = SensorField("market", fm_scheme="fm1").deserialize(post_message["market"])
-    for job, horizon in zip(jobs, horizons):
-        assert job.kwargs["horizon"] == horizon
-        assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon
-        assert job.kwargs["timed_value_type"] == "Price"
-        assert job.kwargs["asset_id"] == market.id
-
-
 @pytest.mark.parametrize(
     "post_message, fm_scheme",
     [
@@ -60,8 +16,7 @@ def test_post_price_data_2_0(db, app, post_message):
 )
 def test_post_prognosis_2_0(db, app, post_message, fm_scheme):
     with app.test_client() as client:
-        # post price data
-        auth_token = get_auth_token(client, "test_supplier@seita.nl", "testtest")
+        auth_token = get_auth_token(client, "test_prosumer@seita.nl", "testtest")
         post_prognosis_response = client.post(
             url_for("flexmeasures_api_v2_0.post_prognosis"),
             json=post_message,

flexmeasures/api/v2_0/tests/test_api_v2_0_sensors_fresh_db.py

@@ -4,7 +4,7 @@
 from flask import url_for
 from iso8601 import parse_date
 
-from flexmeasures.api.common.utils.api_utils import get_generic_asset
+from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.tests.utils import get_auth_token
 from flexmeasures.api.v2_0.tests.utils import (
     message_for_post_price_data,
@@ -15,8 +15,8 @@
 @pytest.mark.parametrize(
     "post_message",
     [
-        message_for_post_price_data(),
-        message_for_post_price_data(prior_instead_of_horizon=True),
+        message_for_post_price_data(market_id=7),
+        message_for_post_price_data(market_id=1, prior_instead_of_horizon=True),
     ],
 )
 def test_post_price_data_2_0(
@@ -46,7 +46,7 @@ def test_post_price_data_2_0(
         assert post_price_data_response.json["type"] == "PostPriceDataResponse"
 
     verify_sensor_data_in_db(
-        post_message, post_message["values"], db, entity_type="market"
+        post_message, post_message["values"], db, entity_type="market", fm_scheme="fm1"
     )
 
     # look for Forecasting jobs in queue
@@ -55,7 +55,7 @@ def test_post_price_data_2_0(
     )  # only one market is affected, but two horizons
     horizons = [timedelta(hours=24), timedelta(hours=48)]
     jobs = sorted(app.queues["forecasting"].jobs, key=lambda x: x.kwargs["horizon"])
-    market = get_generic_asset(post_message["market"], "market")
+    market = SensorField("market", fm_scheme="fm1").deserialize(post_message["market"])
     for job, horizon in zip(jobs, horizons):
         assert job.kwargs["horizon"] == horizon
         assert job.kwargs["start"] == parse_date(post_message["start"]) + horizon

flexmeasures/api/v2_0/tests/utils.py

@@ -32,6 +32,7 @@ def get_asset_post_data() -> dict:
 
 
 def message_for_post_price_data(
+    market_id: int,
     tile_n: int = 1,
     compress_n: int = 1,
     duration: Optional[timedelta] = None,
@@ -59,7 +60,7 @@ def message_for_post_price_data(
         duration=duration,
         invalid_unit=invalid_unit,
     )
-    message["market"] = "ea1.2018-06.localhost:fm1.1"
+    message["market"] = f"ea1.2018-06.localhost:fm1.{market_id}"
     message["horizon"] = duration_isoformat(timedelta(hours=0))
     if no_horizon or prior_instead_of_horizon:
         message.pop("horizon", None)
@@ -144,10 +145,13 @@ def verify_sensor_data_in_db(
 
 
 def message_for_post_prognosis(fm_scheme: str = "fm1"):
+    """
+    Posting prognosis for a wind mill's production.
+    """
     message = {
         "type": "PostPrognosisRequest",
         "connection": f"ea1.2018-06.localhost:{fm_scheme}.2",
-        "values": [300, 300, 300, 0, 0, 300],
+        "values": [-300, -300, -300, 0, 0, -300],
         "start": "2021-01-01T00:00:00Z",
         "duration": "PT1H30M",
         "prior": "2020-12-31T18:00:00Z",

flexmeasures/conftest.py

@@ -24,8 +24,8 @@
 from flexmeasures.data.services.users import create_user
 from flexmeasures.data.models.assets import AssetType, Asset, Power
 from flexmeasures.data.models.data_sources import DataSource
-from flexmeasures.data.models.markets import Market, Price
 from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorType
+from flexmeasures.data.models.markets import Market, MarketType, Price
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.user import User
 
@@ -449,46 +449,57 @@ def add_charging_station_assets(
     }
 
 
-@pytest.fixture(scope="function", autouse=True)
-def add_weather_sensors(db: SQLAlchemy):
+@pytest.fixture(scope="module")
+def add_weather_sensors(db) -> Dict[str, WeatherSensor]:
+    return create_weather_sensors(db)
+
+
+@pytest.fixture(scope="function")
+def add_weather_sensors_fresh_db(fresh_db) -> Dict[str, WeatherSensor]:
+    return create_weather_sensors(fresh_db)
+
+
+def create_weather_sensors(db: SQLAlchemy):
     """Add some weather sensors and weather sensor types."""
 
     test_sensor_type = WeatherSensorType(name="wind_speed")
     db.session.add(test_sensor_type)
-    sensor = WeatherSensor(
+    wind_sensor = WeatherSensor(
         name="wind_speed_sensor",
         weather_sensor_type_name="wind_speed",
         event_resolution=timedelta(minutes=5),
         latitude=33.4843866,
         longitude=126,
         unit="m/s",
     )
-    db.session.add(sensor)
+    db.session.add(wind_sensor)
 
     test_sensor_type = WeatherSensorType(name="temperature")
     db.session.add(test_sensor_type)
-    sensor = WeatherSensor(
+    temp_sensor = WeatherSensor(
         name="temperature_sensor",
         weather_sensor_type_name="temperature",
         event_resolution=timedelta(minutes=5),
         latitude=33.4843866,
         longitude=126.0,
         unit="°C",
     )
-    db.session.add(sensor)
+    db.session.add(temp_sensor)
+    return {"wind": wind_sensor, "temperature": temp_sensor}
 
 
-@pytest.fixture(scope="function", autouse=True)
+@pytest.fixture(scope="module")
 def add_sensors(db: SQLAlchemy):
     """Add some generic sensors."""
     height_sensor = Sensor(
         name="my daughter's height",
         unit="m",
     )
     db.session.add(height_sensor)
+    return height_sensor
 
 
-@pytest.fixture(scope="function", autouse=True)
+@pytest.fixture(scope="function")
 def clean_redis(app):
     failed = app.queues["forecasting"].failed_job_registry
     app.queues["forecasting"].empty()

flexmeasures/data/models/time_series.py

@@ -17,6 +17,9 @@
 )
 from flexmeasures.data.services.time_series import collect_time_series_data
 from flexmeasures.utils.entity_address_utils import build_entity_address
+from flexmeasures.data.models.charts import chart_type_to_chart_specs
+from flexmeasures.utils.time_utils import server_now
+from flexmeasures.utils.flexmeasures_inflection import capitalize
 
 
 class Sensor(db.Model, tb.SensorDBMixin):
@@ -118,7 +121,7 @@ def chart(
 
     @property
     def timerange(self) -> Dict[str, datetime_type]:
-        """Timerange for which sensor data exists.
+        """Time range for which sensor data exists.
 
         :returns: dictionary with start and end, for example:
                   {