Sunset old flex config fields

documentation/api/change_log.rst

@@ -5,6 +5,11 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
+v3.0-6 | 2023-02-01
+"""""""""""""""""""
+
+- Sunset all fields that were moved to ``flex-model`` and ``flex-context`` fields to `/sensors/<id>/schedules/trigger` (POST). See v3.0-5.
+
 v3.0-5 | 2023-01-04
 """""""""""""""""""
 
@@ -21,7 +26,7 @@ v3.0-5 | 2023-01-04
     - ``production-price-sensor`` -> send in ``flex-context`` instead
     - ``inflexible-device-sensors`` -> send in ``flex-context`` instead
 
-- Introduced the ``duration``field to `/sensors/<id>/schedules/trigger` (POST) for setting a planning horizon explicitly.
+- Introduced the ``duration`` field to `/sensors/<id>/schedules/trigger` (POST) for setting a planning horizon explicitly.
 - Allow posting ``soc-targets`` to `/sensors/<id>/schedules/trigger` (POST) that exceed the default planning horizon, and ignore posted targets that exceed the max planning horizon.
 - Added a subsection on deprecating and sunsetting to the Introduction section.
 - Added a subsection on describing flexibility to the Notation section.

documentation/api/notation.rst

@@ -367,7 +367,7 @@ The resolution of the underlying data will remain zero (and the returned message
 .. _sources:
 
 Sources
--------
+^^^^^^^
 
 Requests for data may filter by source. FlexMeasures keeps track of the data source (the data's author, for example, a user, forecaster or scheduler belonging to a given organisation) of time series data.
 For example, to obtain data originating from data source 42, include the following:

documentation/changelog.rst

@@ -6,6 +6,19 @@ FlexMeasures Changelog
 v0.13.0 | February XX, 2023
 ============================
 
+.. warning:: The API endpoint (`[POST] /sensors/(id)/schedules/trigger <api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_) to make new schedules sunsets the deprecated (since v0.12) storage flexibility parameters (they move to the ``flex-model`` parameter group), as well as the parameters describing other sensors (they move to ``flex-context``).
+
+New features
+-------------
+
+Bugfixes
+-----------
+
+
+Infrastructure / Support
+----------------------
+* Sunset several API fields for `/sensors/<id>/schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 <https://www.github.com/FlexMeasures/flexmeasures/pull/580>`_]
+
 
 v0.12.1 | January XX, 2023
 ============================
@@ -15,7 +28,6 @@ Bugfixes
 * Fix validation of (deprecated) API parameter ``roundtrip-efficiency`` [see `PR #582 <https://www.github.com/FlexMeasures/flexmeasures/pull/582>`_]
 
 
-
 v0.12.0 | January 4, 2023
 ============================
 

flexmeasures/api/v3_0/sensors.py

@@ -1,13 +1,12 @@
 from datetime import datetime, timedelta
-from typing import List, Dict, Optional
+from typing import Optional
 
 from flask import current_app
 from flask_classful import FlaskView, route
 from flask_json import as_json
 from flask_security import auth_required
 import isodate
 from marshmallow import fields, ValidationError
-from marshmallow.validate import OneOf
 from rq.job import Job, NoSuchJobError
 from timely_beliefs import BeliefsDataFrame
 from webargs.flaskparser import use_args, use_kwargs
@@ -18,7 +17,6 @@
     unknown_schedule,
     invalid_flex_config,
 )
-from flexmeasures.api.common.utils.deprecation_utils import deprecate_fields
 from flexmeasures.api.common.utils.validators import (
     optional_duration_accepted,
 )
@@ -42,27 +40,13 @@
     get_data_source_for_job,
 )
 from flexmeasures.utils.time_utils import duration_isoformat
-from flexmeasures.utils.unit_utils import ur
 
 
 # Instantiate schemas outside of endpoint logic to minimize response time
 get_sensor_schema = GetSensorDataSchema()
 post_sensor_schema = PostSensorDataSchema()
 sensors_schema = SensorSchema(many=True)
 
-DEPRECATED_FLEX_CONFIGURATION_FIELDS = [
-    "soc-at-start",
-    "soc-min",
-    "soc-max",
-    "soc-unit",
-    "roundtrip-efficiency",
-    "prefer-charging-sooner",
-    "soc-targets",
-    "consumption-price-sensor",
-    "production-price-sensor",
-    "inflexible-device-sensors",
-]
-
 
 class SensorAPI(FlaskView):
 
@@ -211,7 +195,6 @@ def get_data(self, response: dict):
         {"sensor": SensorIdField(data_key="id")},
         location="path",
     )
-    # TODO: Everything other than start_of_schedule, prior, flex_model and flex_context is to be deprecated in 0.13. We let the scheduler decide (flex model) or nest (portfolio)
     @use_kwargs(
         {
             "start_of_schedule": AwareDateTimeField(
@@ -222,36 +205,9 @@ def get_data(self, response: dict):
                 load_default=PlanningDurationField.load_default
             ),
             "flex_model": fields.Dict(data_key="flex-model"),
-            "soc_sensor_id": fields.Str(data_key="soc-sensor", required=False),
-            "roundtrip_efficiency": fields.Raw(data_key="roundtrip-efficiency"),
-            "start_value": fields.Float(data_key="soc-at-start"),
-            "soc_min": fields.Float(data_key="soc-min"),
-            "soc_max": fields.Float(data_key="soc-max"),
-            "unit": fields.Str(
-                data_key="soc-unit",
-                validate=OneOf(
-                    [
-                        "kWh",
-                        "MWh",
-                    ]
-                ),
-            ),  # todo: allow unit to be set per field, using QuantityField("%", validate=validate.Range(min=0, max=1))
-            "targets": fields.List(fields.Dict, data_key="soc-targets"),
-            "prefer_charging_sooner": fields.Bool(
-                data_key="prefer-charging-sooner", required=False
-            ),
             "flex_context": fields.Nested(
                 FlexContextSchema, required=False, data_key="flex-context"
             ),
-            "consumption_price_sensor": SensorIdField(
-                data_key="consumption-price-sensor", required=False
-            ),
-            "production_price_sensor": SensorIdField(
-                data_key="production-price-sensor", required=False
-            ),
-            "inflexible_device_sensors": fields.List(
-                SensorIdField, data_key="inflexible-device-sensors", required=False
-            ),
         },
         location="json",
     )
@@ -261,16 +217,6 @@ def trigger_schedule(  # noqa: C901
         start_of_schedule: datetime,
         duration: timedelta,
         belief_time: Optional[datetime] = None,
-        start_value: Optional[float] = None,
-        soc_min: Optional[float] = None,
-        soc_max: Optional[float] = None,
-        unit: Optional[str] = None,
-        roundtrip_efficiency: Optional[ur.Quantity] = None,
-        prefer_charging_sooner: Optional[bool] = True,
-        consumption_price_sensor: Optional[Sensor] = None,
-        production_price_sensor: Optional[Sensor] = None,
-        inflexible_device_sensors: Optional[List[Sensor]] = None,
-        soc_sensor_id: Optional[int] = None,
         flex_model: Optional[dict] = None,
         flex_context: Optional[dict] = None,
         **kwargs,
@@ -383,69 +329,6 @@ def trigger_schedule(  # noqa: C901
         :status 405: INVALID_METHOD
         :status 422: UNPROCESSABLE_ENTITY
         """
-        # -- begin deprecation logic, can be removed after 0.13
-        deprecate_fields(
-            DEPRECATED_FLEX_CONFIGURATION_FIELDS,
-            deprecation_date="2022-12-14",
-            deprecation_link="https://flexmeasures.readthedocs.io/en/latest/api/change_log.html#v3-0-5-2022-12-30",
-            sunset_date="2023-02-01",
-            sunset_link="https://flexmeasures.readthedocs.io/en/latest/api/change_log.html#v3-0-5-2022-12-30",
-        )
-        found_fields: Dict[str, List[str]] = dict(model=[], context=[])
-        deprecation_message = ""
-        # flex-model
-        for param, param_name in [
-            (start_value, "soc-at-start"),
-            (soc_min, "soc-min"),
-            (soc_max, "soc-max"),
-            (unit, "soc-unit"),
-            (kwargs.get("targets"), "soc-targets"),
-            (roundtrip_efficiency, "roundtrip-efficiency"),
-            (
-                prefer_charging_sooner,
-                "prefer-charging-sooner",
-            ),
-        ]:
-            if flex_model is None:
-                flex_model = {}
-            if param is not None:
-                if param_name not in flex_model:
-                    flex_model[param_name] = param
-                found_fields["model"].append(param_name)
-        # flex-context
-        for param, param_name in [
-            (
-                consumption_price_sensor,
-                "consumption-price-sensor",
-            ),
-            (
-                production_price_sensor,
-                "production-price-sensor",
-            ),
-            (
-                inflexible_device_sensors,
-                "inflexible-device-sensors",
-            ),
-        ]:
-            if flex_context is None:
-                flex_context = {}
-            if param is not None:
-                if param_name not in flex_context:
-                    flex_context[param_name] = param
-                found_fields["context"].append(param_name)
-        if found_fields["model"] or found_fields["context"]:
-            deprecation_message = "The following fields you sent are deprecated and will be sunset in the next version:"
-            if found_fields["model"]:
-                deprecation_message += f" {', '.join(found_fields['model'])} (please pass as part of flex_model)."
-            if found_fields["context"]:
-                deprecation_message += f" {', '.join(found_fields['context'])} (please pass as part of flex_context)."
-
-        if soc_sensor_id is not None:
-            deprecation_message += (
-                "The field soc-sensor-id is be deprecated and will be sunset in v0.13."
-            )
-        # -- end deprecation logic
-
         end_of_schedule = start_of_schedule + duration
         scheduler_kwargs = dict(
             sensor=sensor,
@@ -470,7 +353,7 @@ def trigger_schedule(  # noqa: C901
         db.session.commit()
 
         response = dict(schedule=job.id)
-        d, s = request_processed(deprecation_message)
+        d, s = request_processed()
         return dict(**response, **d), s
 
     @route("/<id>/schedules/<uuid>", methods=["GET"])

flexmeasures/api/v3_0/tests/test_sensor_schedules.py

@@ -76,14 +76,7 @@ def test_trigger_schedule_with_invalid_flexmodel(
 @pytest.mark.parametrize(
     "message, asset_name",
     [
-        (message_for_trigger_schedule(deprecated_format_pre012=True), "Test battery"),
         (message_for_trigger_schedule(), "Test battery"),
-        (
-            message_for_trigger_schedule(
-                with_targets=True, deprecated_format_pre012=True
-            ),
-            "Test charging station",
-        ),
         (message_for_trigger_schedule(with_targets=True), "Test charging station"),
     ],
 )
@@ -101,9 +94,6 @@ def test_trigger_and_get_schedule(
     assert len(app.queues["scheduling"]) == 0
 
     sensor = Sensor.query.filter(Sensor.name == asset_name).one_or_none()
-    # This makes sure we have fresh data. A hack we can remove after the deprecation cases are removed.
-    TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id).delete()
-
     with app.test_client() as client:
         auth_token = get_auth_token(client, "test_prosumer_user@seita.nl", "testtest")
         trigger_schedule_response = client.post(
@@ -112,11 +102,6 @@ def test_trigger_and_get_schedule(
             headers={"Authorization": auth_token},
         )
         print("Server responded with:\n%s" % trigger_schedule_response.json)
-        if "flex-model" not in message:
-            check_deprecation(trigger_schedule_response)
-            assert (
-                "soc-min" in trigger_schedule_response.json["message"]
-            )  # deprecation warning
         assert trigger_schedule_response.status_code == 200
         job_id = trigger_schedule_response.json["schedule"]
 

flexmeasures/api/v3_0/tests/utils.py

@@ -45,7 +45,6 @@ def message_for_trigger_schedule(
     with_targets: bool = False,
     realistic_targets: bool = True,
     too_far_into_the_future_targets: bool = False,
-    deprecated_format_pre012: bool = False,
 ) -> dict:
     message = {
         "start": "2015-01-01T00:00:00+01:00",
@@ -56,18 +55,13 @@ def message_for_trigger_schedule(
             "start"
         ] = "2040-01-01T00:00:00+01:00"  # We have no beliefs in our test database about 2040 prices
 
-    flex_model = {
+    message["flex-model"] = {
         "soc-at-start": 12.1,  # in kWh, according to soc-unit
         "soc-min": 0,  # in kWh, according to soc-unit
         "soc-max": 40,  # in kWh, according to soc-unit
         "soc-unit": "kWh",
         "roundtrip-efficiency": "98%",
     }
-    if deprecated_format_pre012:
-        # unpack flex model directly as message fields
-        message = dict(**message, **flex_model)
-    else:
-        message["flex-model"] = flex_model
     if with_targets:
         if realistic_targets:
             # this target (in kWh, according to soc-unit) is well below the soc_max_in_mwh on the battery's sensor attributes
@@ -80,9 +74,7 @@ def message_for_trigger_schedule(
             target_datetime = "2015-02-02T23:00:00+01:00"
         else:
             target_datetime = "2015-01-02T23:00:00+01:00"
-        targets = [{"value": target_value, "datetime": target_datetime}]
-        if deprecated_format_pre012:
-            message["soc-targets"] = targets
-        else:
-            message["flex-model"]["soc-targets"] = targets
+        message["flex-model"]["soc-targets"] = [
+            {"value": target_value, "datetime": target_datetime}
+        ]
     return message