400 support passing a custom duration to /schedules/trigger [POST]

documentation/api/change_log.rst

@@ -13,13 +13,15 @@ v3.0-5 | 2022-12-30
     - ``soc-at-start`` -> send in ``flex-model`` instead
     - ``soc-min`` -> send in ``flex-model`` instead
     - ``soc-max`` -> send in ``flex-model`` instead
+    - ``soc-targets`` -> send in ``flex-model`` instead
     - ``soc-unit`` -> send in ``flex-model`` instead
     - ``roundtrip-efficiency`` -> send in ``flex-model`` instead
     - ``prefer-charging-sooner`` -> send in ``flex-model`` instead
     - ``consumption-price-sensor`` -> send in ``flex-context`` instead
     - ``production-price-sensor`` -> send in ``flex-context`` instead
     - ``inflexible-device-sensors`` -> send in ``flex-context`` instead
 
+- Allow posting ``soc-targets`` to `/sensors/<id>/schedules/trigger` (POST) that exceed the default planning horizon, but not the max planning horizon.
 - Added a subsection on deprecating and sunsetting to the Introduction section.
 - Added a subsection on describing flexibility to the Notation section.
 

documentation/configuration.rst

@@ -253,9 +253,20 @@ Default: ``timedelta(days=7)``
 FLEXMEASURES_PLANNING_HORIZON
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The horizon to use when making schedules.
+The default horizon for making schedules.
+API users can set a custom duration if they need to.
 
-Default: ``timedelta(hours=2 * 24)``
+Default: ``timedelta(days=2)``
+
+
+FLEXMEASURES_MAX_PLANNING_HORIZON
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The maximum horizon for making schedules.
+API users are not able to request longer schedules.
+Set to ``None`` to forgo this limitation.
+
+Default: ``timedelta(days=7, hours=1)``
 
 
 Access Tokens

flexmeasures/api/v3_0/sensors.py

@@ -34,7 +34,7 @@
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField
-from flexmeasures.data.schemas.times import AwareDateTimeField
+from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField
 from flexmeasures.data.schemas.units import QuantityField
 from flexmeasures.data.schemas.scheduling import FlexContextSchema
 from flexmeasures.data.services.sensors import get_sensors
@@ -220,6 +220,9 @@ def get_data(self, response: dict):
                 data_key="start", format="iso", required=True
             ),
             "belief_time": AwareDateTimeField(format="iso", data_key="prior"),
+            "duration": PlanningDurationField(
+                load_default=PlanningDurationField.load_default
+            ),
             "flex_model": fields.Dict(data_key="flex-model"),
             "soc_sensor_id": fields.Str(data_key="soc-sensor", required=False),
             "roundtrip_efficiency": QuantityField(
@@ -262,6 +265,7 @@ def trigger_schedule(  # noqa: C901
         self,
         sensor: Sensor,
         start_of_schedule: datetime,
+        duration: timedelta,
         belief_time: Optional[datetime] = None,
         start_value: Optional[float] = None,
         soc_min: Optional[float] = None,
@@ -298,9 +302,11 @@ def trigger_schedule(  # noqa: C901
                   See https://github.com/FlexMeasures/flexmeasures/issues/485. Until then, it is possible to call this endpoint for one flexible endpoint at a time
                   (considering already scheduled sensors as inflexible).
 
-        The length of schedules is set by the config setting :ref:`planning_horizon_config`, defaulting to 12 hours.
-
-        .. todo:: add a schedule duration parameter, instead of always falling back to FLEXMEASURES_PLANNING_HORIZON
+        The length of the schedule can be set explicitly through the 'duration' field.
+        Otherwise, it is set by the config setting :ref:`planning_horizon_config`, which defaults to 48 hours.
+        If the flex-model contains targets that lie beyond the planning horizon, the length of the schedule is extended to accommodate them.
+        Finally, the schedule length is limited by :ref:`max_planning_horizon_config`, which defaults to 169 hours.
+        Targets that exceed the max planning horizon are not accepted.
 
         The appropriate algorithm is chosen by FlexMeasures (based on asset type).
         It's also possible to use custom schedulers and custom flexibility models, see :ref:`plugin_customization`.
@@ -323,8 +329,8 @@ def trigger_schedule(  # noqa: C901
 
         **Example request B**
 
-        This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh,
-        with a target state of charge of 25 kWh at 4.00pm.
+        This message triggers a 24-hour schedule for a storage asset, starting at 10.00am,
+        at which the state of charge (soc) is 12.1 kWh, with a target state of charge of 25 kWh at 4.00pm.
         The minimum and maximum soc are set to 10 and 25 kWh, respectively.
         Roundtrip efficiency for use in scheduling is set to 98%.
         Aggregate consumption (of all devices within this EMS) should be priced by sensor 9,
@@ -337,6 +343,7 @@ def trigger_schedule(  # noqa: C901
 
             {
                 "start": "2015-06-02T10:00:00+00:00",
+                "duration": "PT24H",
                 "flex-model": {
                     "soc-at-start": 12.1,
                     "soc-unit": "kWh",
@@ -447,9 +454,7 @@ def trigger_schedule(  # noqa: C901
             )
         # -- end deprecation logic
 
-        end_of_schedule = start_of_schedule + current_app.config.get(  # type: ignore
-            "FLEXMEASURES_PLANNING_HORIZON"
-        )
+        end_of_schedule = start_of_schedule + duration
         scheduler_kwargs = dict(
             sensor=sensor,
             start=start_of_schedule,

flexmeasures/api/v3_0/tests/test_sensor_schedules.py

@@ -1,6 +1,6 @@
 from flask import url_for
 import pytest
-from isodate import parse_datetime
+from isodate import parse_datetime, parse_duration
 
 import pandas as pd
 from rq.job import Job
@@ -28,6 +28,14 @@
             "Not a valid number",
         ),
         (message_for_trigger_schedule(), "soc-unit", "MWH", "Must be one of"),
+        (
+            message_for_trigger_schedule(
+                with_targets=True, too_far_into_the_future_targets=True
+            ),
+            "soc-targets",
+            None,
+            "Target datetime exceeds",
+        ),
     ],
 )
 def test_trigger_schedule_with_invalid_flexmodel(
@@ -127,6 +135,31 @@ def test_trigger_and_get_schedule(
         is True
     )
 
+    # Derive some expectations from the POSTed message
+    if "flex-model" not in message:
+        start_soc = message["soc-at-start"] / 1000  # in MWh
+        roundtrip_efficiency = message["roundtrip-efficiency"]
+        soc_targets = message.get("soc-targets")
+    else:
+        start_soc = message["flex-model"]["soc-at-start"] / 1000  # in MWh
+        roundtrip_efficiency = message["flex-model"]["roundtrip-efficiency"]
+        soc_targets = message["flex-model"].get("soc-targets")
+    resolution = sensor.event_resolution
+    if soc_targets:
+        # Schedule length may be extended to accommodate targets that lie beyond the schedule's end
+        max_target_datetime = max(
+            [parse_datetime(soc_target["datetime"]) for soc_target in soc_targets]
+        )
+        expected_length_of_schedule = (
+            max(
+                parse_duration(message["duration"]),
+                max_target_datetime - parse_datetime(message["start"]),
+            )
+            / resolution
+        )
+    else:
+        expected_length_of_schedule = parse_duration(message["duration"]) / resolution
+
     # check results are in the database
 
     # First, make sure the scheduler data source is now there
@@ -140,24 +173,11 @@ def test_trigger_and_get_schedule(
         .filter(TimedBelief.source_id == scheduler_source.id)
         .all()
     )
-    resolution = sensor.event_resolution
     consumption_schedule = pd.Series(
         [-v.event_value for v in power_values],
         index=pd.DatetimeIndex([v.event_start for v in power_values], freq=resolution),
     )  # For consumption schedules, positive values denote consumption. For the db, consumption is negative
-    assert (
-        len(consumption_schedule)
-        == app.config.get("FLEXMEASURES_PLANNING_HORIZON") / resolution
-    )
-
-    if "flex-model" not in message:
-        start_soc = message["soc-at-start"] / 1000  # in MWh
-        roundtrip_efficiency = message["roundtrip-efficiency"]
-        soc_targets = message.get("soc-targets")
-    else:
-        start_soc = message["flex-model"]["soc-at-start"] / 1000  # in MWh
-        roundtrip_efficiency = message["flex-model"]["roundtrip-efficiency"]
-        soc_targets = message["flex-model"].get("soc-targets")
+    assert len(consumption_schedule) == expected_length_of_schedule
 
     # check targets, if applicable
     if soc_targets:
@@ -187,7 +207,7 @@ def test_trigger_and_get_schedule(
     print("Server responded with:\n%s" % get_schedule_response.json)
     assert get_schedule_response.status_code == 200
     # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse"
-    assert len(get_schedule_response.json["values"]) == 192
+    assert len(get_schedule_response.json["values"]) == expected_length_of_schedule
 
     # Test that a shorter planning horizon yields the same result for the shorter planning horizon
     get_schedule_message["duration"] = "PT6H"

flexmeasures/api/v3_0/tests/utils.py

@@ -44,10 +44,12 @@ def message_for_trigger_schedule(
     unknown_prices: bool = False,
     with_targets: bool = False,
     realistic_targets: bool = True,
+    too_far_into_the_future_targets: bool = False,
     deprecated_format_pre012: bool = False,
 ) -> dict:
     message = {
         "start": "2015-01-01T00:00:00+01:00",
+        "duration": "PT24H",  # Will be extended in case of targets that would otherwise lie beyond the schedule's end
     }
     if unknown_prices:
         message[
@@ -69,10 +71,16 @@ def message_for_trigger_schedule(
     if with_targets:
         if realistic_targets:
             # this target (in kWh, according to soc-unit) is well below the soc_max_in_mwh on the battery's sensor attributes
-            targets = [{"value": 25, "datetime": "2015-01-02T23:00:00+01:00"}]
+            target_value = 25
         else:
             # this target (in kWh, according to soc-unit) is actually higher than soc_max_in_mwh on the battery's sensor attributes
-            targets = [{"value": 25000, "datetime": "2015-01-02T23:00:00+01:00"}]
+            target_value = 25000
+        if too_far_into_the_future_targets:
+            # this target exceeds FlexMeasures' default max planning horizon
+            target_datetime = "2015-02-02T23:00:00+01:00"
+        else:
+            target_datetime = "2015-01-02T23:00:00+01:00"
+        targets = [{"value": target_value, "datetime": target_datetime}]
         if deprecated_format_pre012:
             message["soc-targets"] = targets
         else:

flexmeasures/data/models/planning/storage.py

@@ -250,11 +250,33 @@ def deserialize_flex_config(self):
         self.ensure_soc_min_max()
 
         # Now it's time to check if our flex configurations holds up to schemas
-        self.flex_model = StorageFlexModelSchema().load(self.flex_model)
+        self.flex_model = StorageFlexModelSchema(self.start).load(self.flex_model)
         self.flex_context = FlexContextSchema().load(self.flex_context)
 
+        # Extend schedule period in case a target exceeds its end
+        self.possibly_extend_end()
+
         return self.flex_model
 
+    def possibly_extend_end(self):
+        """Extend schedule period in case a target exceeds its end.
+
+        The schedule's duration is possibly limited by the server config setting 'FLEXMEASURES_MAX_PLANNING_HORIZON'.
+        """
+        soc_targets = self.flex_model.get("soc_targets")
+        if soc_targets:
+            max_target_datetime = max(
+                [soc_target["datetime"] for soc_target in soc_targets]
+            )
+            if max_target_datetime > self.end:
+                max_server_horizon = current_app.config.get(
+                    "FLEXMEASURES_MAX_PLANNING_HORIZON"
+                )
+                if max_server_horizon:
+                    self.end = min(max_target_datetime, self.start + max_server_horizon)
+                else:
+                    self.end = max_target_datetime
+
     def get_min_max_targets(
         self, deserialized_names: bool = True
     ) -> tuple[float | None, float | None]:
@@ -311,7 +333,7 @@ def ensure_soc_min_max(self):
 
 
 def build_device_soc_targets(
-    targets: List[Dict[datetime, float]] | pd.Series,
+    targets: List[Dict[str, datetime | float]] | pd.Series,
     soc_at_start: float,
     start_of_schedule: datetime,
     end_of_schedule: datetime,
@@ -334,7 +356,7 @@ def build_device_soc_targets(
     TODO: this function could become the deserialization method of a new SOCTargetsSchema (targets, plural), which wraps SOCTargetSchema.
 
     """
-    if isinstance(targets, pd.Series):  # some teats prepare it this way
+    if isinstance(targets, pd.Series):  # some tests prepare it this way
         device_targets = targets
     else:
         device_targets = initialize_series(
@@ -352,7 +374,7 @@ def build_device_soc_targets(
             )  # otherwise DST would be problematic
             if target_datetime > end_of_schedule:
                 raise ValueError(
-                    f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")}.'
+                    f'Target datetime exceeds {end_of_schedule}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_MAX_PLANNING_HORIZON")}.'
                 )
 
             device_targets.loc[target_datetime] = target_value

flexmeasures/data/schemas/scheduling/storage.py

@@ -1,5 +1,10 @@
-from marshmallow import Schema, post_load, validate, fields
-from marshmallow.validate import OneOf
+from __future__ import annotations
+
+from datetime import datetime
+
+from flask import current_app
+from marshmallow import Schema, post_load, validate, validates, fields
+from marshmallow.validate import OneOf, ValidationError
 
 from flexmeasures.data.schemas.times import AwareDateTimeField
 from flexmeasures.data.schemas.units import QuantityField
@@ -42,7 +47,26 @@ class StorageFlexModelSchema(Schema):
     )
     prefer_charging_sooner = fields.Bool(data_key="prefer-charging-sooner")
 
-    @post_load()
+    def __init__(self, start: datetime, *args, **kwargs):
+        """Pass the schedule's start, so we can use it to validate soc-target datetimes."""
+        self.start = start
+        super().__init__(*args, **kwargs)
+
+    @validates("soc_targets")
+    def check_whether_targets_exceed_max_planning_horizon(
+        self, soc_targets: list[dict[str, datetime | float]]
+    ):
+        if not soc_targets:
+            return
+        max_server_horizon = current_app.config.get("FLEXMEASURES_MAX_PLANNING_HORIZON")
+        max_target_datetime = max([target["datetime"] for target in soc_targets])
+        max_server_datetime = self.start + max_server_horizon
+        if max_target_datetime > max_server_datetime:
+            raise ValidationError(
+                f'Target datetime exceeds {max_server_datetime}. Maximum scheduling horizon is {current_app.config.get("FLEXMEASURES_MAX_PLANNING_HORIZON")}.'
+            )
+
+    @post_load
     def post_load_sequence(self, data: dict, **kwargs) -> dict:
         """Perform some checks and corrections after we loaded."""
         # currently we only handle MWh internally

flexmeasures/data/schemas/times.py

@@ -1,6 +1,7 @@
 from typing import Union, Optional
 from datetime import datetime, timedelta
 
+from flask import current_app
 from marshmallow import fields
 import isodate
 from isodate.isoerror import ISO8601Error
@@ -64,6 +65,15 @@ def ground_from(
         return duration
 
 
+class PlanningDurationField(DurationField):
+    @classmethod
+    def load_default(cls):
+        """
+        Use this with the load_default arg to __init__ if you want the default FlexMeasures planning horizon.
+        """
+        return current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")
+
+
 class AwareDateTimeField(MarshmallowClickMixin, fields.AwareDateTime):
     """Field that de-serializes to a timezone aware datetime
     and serializes back to a string."""

flexmeasures/utils/config_defaults.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import timedelta
 import logging
 from typing import List, Optional, Union, Dict, Tuple
@@ -112,7 +114,8 @@ class Config(object):
     }  # how to group assets by asset types
     FLEXMEASURES_LP_SOLVER: str = "cbc"
     FLEXMEASURES_JOB_TTL: timedelta = timedelta(days=1)
-    FLEXMEASURES_PLANNING_HORIZON: timedelta = timedelta(hours=2 * 24)
+    FLEXMEASURES_PLANNING_HORIZON: timedelta = timedelta(days=2)
+    FLEXMEASURES_MAX_PLANNING_HORIZON: timedelta | None = timedelta(days=7, hours=1)
     FLEXMEASURES_PLANNING_TTL: timedelta = timedelta(
         days=7
     )  # Time to live for UDI event ids of successful scheduling jobs. Set a negative timedelta to persist forever.