From ce8169cbd6866f70ed45e094749fa1b7ac20ae85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sat, 1 Oct 2022 18:29:28 +0200
Subject: [PATCH 01/44] Better documentation of flexibility model for storage
 in endpoint; refactor its parameters and handling within the code for
 readability
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/api/v3_0/sensors.py              | 55 +++++++++----
 flexmeasures/data/models/planning/battery.py  | 30 ++-----
 .../data/models/planning/charging_station.py  | 29 ++-----
 flexmeasures/data/models/planning/utils.py    | 78 +++++++++++++++++++
 flexmeasures/data/services/scheduling.py      | 77 +++++++-----------
 .../tests/test_scheduling_jobs_fresh_db.py    |  3 +-
 6 files changed, 163 insertions(+), 109 deletions(-)

diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index 35f0248b8..0cf7076ee 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -204,7 +204,7 @@ def get_data(self, response: dict):
                 validate=validate.Range(min=0, max=1),
                 data_key="roundtrip-efficiency",
             ),
-            "value": fields.Float(data_key="soc-at-start"),
+            "start_value": fields.Float(data_key="soc-at-start"),
             "soc_min": fields.Float(data_key="soc-min"),
             "soc_max": fields.Float(data_key="soc-max"),
             "start_of_schedule": AwareDateTimeField(
@@ -251,11 +251,35 @@ def trigger_schedule(  # noqa: C901
 
         .. :quickref: Schedule; Trigger scheduling job
 
-        The message should contain a flexibility model.
+        Trigger FlexMeasures to create a schedule for this sensor.
+        The assumption is that this sensor is the energy sensor on a flexible asset.
+
+        In this request, you can describe:
+
+        - the schedule (start, unit, prior)
+        - the flexibility model for the sensor (see below, only storage models are supported at the moment)
+        - the EMS the sensor operates in (inflexible device sensors, sensors which put a price on consumption and/or production)
+
+        Note: This endpoint does not support an EMS with multiple flexible sensors. This will happen in another endpoint.
+              See https://github.com/FlexMeasures/flexmeasures/issues/485
+
+        Flexibility models apply to the sensor's asset type:
+
+        1) For storage sensors (e.g. battery, charging stations), the schedule deals with the state of charge (SOC).
+           The possible flexibility parameters are:
+
+            - soc-at-start (defaults to 0)
+            - soc-unit (kWh or MWh)
+            - soc-min (defaults to 0)
+            - soc-max (defaults to max soc target)
+            - soc-targets (defaults to NaN values)
+            - roundtrip-efficiency (defaults to 100%)
+
+        2) Heat pump sensors are work in progress.
 
         **Example request A**
 
-        This message triggers a schedule starting at 10.00am, at which the state of charge (soc) is 12.1 kWh.
+        This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh.
 
         .. code-block:: json
 
@@ -324,19 +348,19 @@ def trigger_schedule(  # noqa: C901
         # todo: if a soc-sensor entity address is passed, persist those values to the corresponding sensor
         #       (also update the note in posting_data.rst about flexibility states not being persisted).
 
-        # get value
-        if "value" not in kwargs:
+        # get starting value
+        if "start_value" not in kwargs:
             return ptus_incomplete()
         try:
-            value = float(kwargs.get("value"))  # type: ignore
+            start_value = float(kwargs.get("start_value"))  # type: ignore
         except ValueError:
             extra_info = "Request includes empty or ill-formatted value(s)."
             current_app.logger.warning(extra_info)
             return ptus_incomplete(extra_info)
         if unit == "kWh":
-            value = value / 1000.0
+            start_value = start_value / 1000.0
 
-        # Convert round-trip efficiency to dimensionless
+        # Convert round-trip efficiency to dimensionless (to the [0,1] range)
         if roundtrip_efficiency is not None:
             roundtrip_efficiency = roundtrip_efficiency.to(
                 ur.Quantity("dimensionless")
@@ -345,6 +369,7 @@ def trigger_schedule(  # noqa: C901
         # get optional min and max SOC
         soc_min = kwargs.get("soc_min", None)
         soc_max = kwargs.get("soc_max", None)
+        # TODO: review when we moved away from capacity having to be described in MWh
         if soc_min is not None and unit == "kWh":
             soc_min = soc_min / 1000.0
         if soc_max is not None and unit == "kWh":
@@ -361,7 +386,7 @@ def trigger_schedule(  # noqa: C901
                 start_of_schedule, end_of_schedule, freq=resolution, closed="right"
             ),  # note that target values are indexed by their due date (i.e. closed="right")
         )
-        # todo: move deserialization of targets into TargetSchema
+        # todo: move this deserialization of targets into newly-created ScheduleTargetSchema
         for target in kwargs.get("targets", []):
 
             # get target value
@@ -411,11 +436,13 @@ def trigger_schedule(  # noqa: C901
             end_of_schedule,
             resolution=resolution,
             belief_time=prior,  # server time if no prior time was sent
-            soc_at_start=value,
-            soc_targets=soc_targets,
-            soc_min=soc_min,
-            soc_max=soc_max,
-            roundtrip_efficiency=roundtrip_efficiency,
+            storage_specs=dict(
+                soc_at_start=start_value,
+                soc_targets=soc_targets,
+                soc_min=soc_min,
+                soc_max=soc_max,
+                roundtrip_efficiency=roundtrip_efficiency,
+            ),
             consumption_price_sensor=consumption_price_sensor,
             production_price_sensor=production_price_sensor,
             inflexible_device_sensors=inflexible_device_sensors,
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index d6e3380e0..c654f15fd 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -20,11 +20,7 @@ def schedule_battery(
     start: datetime,
     end: datetime,
     resolution: timedelta,
-    soc_at_start: float,
-    soc_targets: Optional[pd.Series] = None,
-    soc_min: Optional[float] = None,
-    soc_max: Optional[float] = None,
-    roundtrip_efficiency: Optional[float] = None,
+    storage_specs: dict,
     prefer_charging_sooner: bool = True,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
@@ -37,6 +33,12 @@ def schedule_battery(
     For the resulting consumption schedule, consumption is defined as positive values.
     """
 
+    soc_at_start = storage_specs.get("soc_at_start")
+    soc_targets = storage_specs.get("soc_targets")
+    soc_min = storage_specs.get("soc_min")
+    soc_max = storage_specs.get("soc_max")
+    roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+
     # Check for required Sensor attributes
     sensor.check_required_attributes(
         [
@@ -46,19 +48,6 @@ def schedule_battery(
         ],
     )
 
-    # Check for round-trip efficiency
-    if roundtrip_efficiency is None:
-        # Get default from sensor, or use 100% otherwise
-        roundtrip_efficiency = sensor.get_attribute("roundtrip_efficiency", 1)
-    if roundtrip_efficiency <= 0 or roundtrip_efficiency > 1:
-        raise ValueError("roundtrip_efficiency expected within the interval (0, 1]")
-
-    # Check for min and max SOC, or get default from sensor
-    if soc_min is None:
-        soc_min = sensor.get_attribute("min_soc_in_mwh")
-    if soc_max is None:
-        soc_max = sensor.get_attribute("max_soc_in_mwh")
-
     # Check for known prices or price forecasts, trimming planning window accordingly
     up_deviation_prices, (start, end) = get_prices(
         (start, end),
@@ -77,12 +66,9 @@ def schedule_battery(
         allow_trimmed_query_window=True,
     )
 
+    soc_targets = soc_targets.tz_convert("UTC")
     start = pd.Timestamp(start).tz_convert("UTC")
     end = pd.Timestamp(end).tz_convert("UTC")
-    if soc_targets is not None:
-        # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot
-        soc_targets = soc_targets.tz_convert("UTC")
-        soc_targets = soc_targets[start + resolution : end]
 
     # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
     # We penalise the future with at most 1 per thousand times the price spread.
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 40a321c63..25461a757 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -20,11 +20,7 @@ def schedule_charging_station(
     start: datetime,
     end: datetime,
     resolution: timedelta,
-    soc_at_start: float,
-    soc_targets: pd.Series,
-    soc_min: Optional[float] = None,
-    soc_max: Optional[float] = None,
-    roundtrip_efficiency: Optional[float] = None,
+    storage_specs: dict,
     prefer_charging_sooner: bool = True,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
@@ -38,24 +34,15 @@ def schedule_charging_station(
     Todo: handle uni-directional charging by setting the "min" or "derivative min" constraint to 0
     """
 
+    soc_at_start = storage_specs.get("soc_at_start")
+    soc_targets = storage_specs.get("soc_targets")
+    soc_min = storage_specs.get("soc_min")
+    soc_max = storage_specs.get("soc_max")
+    roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+
     # Check for required Sensor attributes
     sensor.check_required_attributes([("capacity_in_mw", (float, int))])
 
-    # Check for round-trip efficiency
-    if roundtrip_efficiency is None:
-        # Get default from sensor, or use 100% otherwise
-        roundtrip_efficiency = sensor.get_attribute("roundtrip_efficiency", 1)
-    if roundtrip_efficiency <= 0 or roundtrip_efficiency > 1:
-        raise ValueError("roundtrip_efficiency expected within the interval (0, 1]")
-
-    # Check for min and max SOC, or get default from sensor
-    if soc_min is None:
-        # Can't drain the EV battery by more than it contains
-        soc_min = sensor.get_attribute("min_soc_in_mwh", 0)
-    if soc_max is None:
-        # Lacking information about the battery's nominal capacity, we use the highest target value as the maximum state of charge
-        soc_max = sensor.get_attribute("max_soc_in_mwh", max(soc_targets.values))
-
     # Check for known prices or price forecasts, trimming planning window accordingly
     up_deviation_prices, (start, end) = get_prices(
         (start, end),
@@ -74,11 +61,9 @@ def schedule_charging_station(
         allow_trimmed_query_window=True,
     )
 
-    # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot
     soc_targets = soc_targets.tz_convert("UTC")
     start = pd.Timestamp(start).tz_convert("UTC")
     end = pd.Timestamp(end).tz_convert("UTC")
-    soc_targets = soc_targets[start + resolution : end]
 
     # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
     # We penalise the future with at most 1 per thousand times the price spread.
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 23a9c347e..5ab769174 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -42,6 +42,84 @@ def initialize_index(
     return i
 
 
+def ensure_storage_specs(
+    specs: Optional[dict],
+    sensor_id: Sensor,
+    start_of_schedule: datetime,
+    end_of_schedule: datetime,
+    resolution: timedelta,
+) -> dict:
+    """
+    Check storage specs and fill in values from context, if possible.
+
+    Storage specs are:
+    - soc_at_start
+    - soc_min
+    - soc_max
+    - soc_targets
+    - roundtrip_efficiency
+    """
+    if specs is None:
+        specs = {}
+
+    sensor: Optional[Sensor] = None
+
+    def ensure_sensor_is_set(sensor) -> Sensor:
+        if sensor is None:
+            sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
+        return sensor
+
+    # Check state of charge
+    # Preferably, a starting soc is given.
+    # Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
+    # Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
+    # and without soc targets and limits the starting soc doesn't matter).
+    if "soc_at_start" not in specs:
+        sensor = ensure_sensor_is_set(sensor)
+        if (
+            start_of_schedule == sensor.get_attribute("soc_datetime")
+            and sensor.get_attribute("soc_in_mwh") is not None
+        ):
+            specs["soc_at_start"] = sensor.get_attribute("soc_in_mwh")
+        else:
+            specs["soc_at_start"] = 0
+
+    # init targets
+    if "soc_targets" not in specs:
+        specs["soc_targets"] = pd.Series(
+            np.nan,
+            index=pd.date_range(
+                start_of_schedule, end_of_schedule, freq=resolution, closed="right"
+            ),
+        )
+    # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot
+    specs["soc_targets"] = specs["soc_targets"][
+        start_of_schedule + resolution : end_of_schedule
+    ]
+
+    # Check for min and max SOC, or get default from sensor
+    if "soc_min" not in specs:
+        sensor = ensure_sensor_is_set(sensor)
+        # Can't drain the EV battery by more than it contains
+        specs["soc_min"] = sensor.get_attribute("min_soc_in_mwh", 0)
+    if "soc_max" not in specs:
+        sensor = ensure_sensor_is_set(sensor)
+        # Lacking information about the battery's nominal capacity, we use the highest target value as the maximum state of charge
+        specs["soc_max"] = sensor.get_attribute(
+            "max_soc_in_mwh", max(specs["soc_targets"].values)
+        )
+
+    # Check for round-trip efficiency
+    if "roundtrip_efficiency" not in specs:
+        sensor = ensure_sensor_is_set(sensor)
+        # Get default from sensor, or use 100% otherwise
+        specs["roundtrip_efficiency"] = sensor.get_attribute("roundtrip_efficiency", 1)
+    if specs["roundtrip_efficiency"] <= 0 or specs["roundtrip_efficiency"] > 1:
+        raise ValueError("roundtrip_efficiency expected within the interval (0, 1]")
+
+    return specs
+
+
 def add_tiny_price_slope(
     prices: pd.DataFrame, col_name: str = "event_value", d: float = 10**-3
 ) -> pd.DataFrame:
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 0e9d2524b..00e1b57e2 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -7,8 +7,6 @@
 
 from flask import current_app
 import click
-import numpy as np
-import pandas as pd
 from rq import get_current_job
 from rq.job import Job
 import timely_beliefs as tb
@@ -16,6 +14,7 @@
 from flexmeasures.data import db
 from flexmeasures.data.models.planning.battery import schedule_battery
 from flexmeasures.data.models.planning.charging_station import schedule_charging_station
+from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.utils import get_data_source, save_to_db
 
@@ -27,8 +26,9 @@
    This might re-enqueue the job or try a different model (which creates a new job).
 """
 
-
-DEFAULT_RESOLUTION = timedelta(minutes=15)
+DEFAULT_RESOLUTION = timedelta(
+    minutes=15
+)  # make_schedule can also fallback to sensor.event_resolution, trigger_schedule also uses that
 
 
 def create_scheduling_job(
@@ -37,18 +37,19 @@ def create_scheduling_job(
     end_of_schedule: datetime,
     belief_time: datetime,
     resolution: timedelta = DEFAULT_RESOLUTION,
-    soc_at_start: Optional[float] = None,
-    soc_targets: Optional[pd.Series] = None,
-    soc_min: Optional[float] = None,
-    soc_max: Optional[float] = None,
-    roundtrip_efficiency: Optional[float] = None,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
     inflexible_device_sensors: Optional[List[Sensor]] = None,
     job_id: Optional[str] = None,
     enqueue: bool = True,
+    storage_specs: Optional[dict] = None,
 ) -> Job:
-    """Supporting quick retrieval of the scheduling job, the job id is the unique entity address of the UDI event.
+    """
+    Create a new Job, which is queued for later execution.
+
+    Before enqueing, we perform some checks on sensor type and specs, for errors we want to bubble up early.
+
+    To support quick retrieval of the scheduling job, the job id is the unique entity address of the UDI event.
     That means one event leads to one job (i.e. actions are event driven).
 
     Target SOC values should be indexed by their due date. For example, for quarter-hourly targets between 5 and 6 AM:
@@ -60,6 +61,10 @@ def create_scheduling_job(
         2010-01-01 06:00:00    3.0
         Freq: 15T, dtype: float64
     """
+    storage_specs = ensure_storage_specs(
+        storage_specs, sensor_id, start_of_schedule, end_of_schedule, resolution
+    )
+
     job = Job.create(
         make_schedule,
         kwargs=dict(
@@ -68,15 +73,11 @@ def create_scheduling_job(
             end=end_of_schedule,
             belief_time=belief_time,
             resolution=resolution,
-            soc_at_start=soc_at_start,
-            soc_targets=soc_targets,
-            soc_min=soc_min,
-            soc_max=soc_max,
-            roundtrip_efficiency=roundtrip_efficiency,
+            storage_specs=storage_specs,
             consumption_price_sensor=consumption_price_sensor,
             production_price_sensor=production_price_sensor,
             inflexible_device_sensors=inflexible_device_sensors,
-        ),
+        ),  # TODO: maybe also pass these sensors as IDs, to avoid potential db sessions confusion
         id=job_id,
         connection=current_app.queues["scheduling"].connection,
         ttl=int(
@@ -101,50 +102,32 @@ def make_schedule(
     end: datetime,
     belief_time: datetime,
     resolution: timedelta,
-    soc_at_start: Optional[float] = None,
-    soc_targets: Optional[pd.Series] = None,
-    soc_min: Optional[float] = None,
-    soc_max: Optional[float] = None,
-    roundtrip_efficiency: Optional[float] = None,
+    storage_specs: Optional[dict],
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
     inflexible_device_sensors: Optional[List[Sensor]] = None,
 ) -> bool:
-    """Preferably, a starting soc is given.
-    Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
-    Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
-    and without soc targets and limits the starting soc doesn't matter).
+    """
+    This function is meant to be queued as a job.
+    It thus potentially runs on a different FlexMeasures node than where the job is created.
+
+    - Choose which scheduling function can be used
+    - Compute schedule
+    - Turn schedukled values into beliefs and save them to db
     """
     # https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
     db.engine.dispose()
 
-    rq_job = get_current_job()
-
-    # find sensor
     sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
+    data_source_name = "Seita"
 
+    rq_job = get_current_job()
     if rq_job:
         click.echo(
             "Running Scheduling Job %s: %s, from %s to %s"
             % (rq_job.id, sensor, start, end)
         )
 
-    if soc_at_start is None:
-        if (
-            start == sensor.get_attribute("soc_datetime")
-            and sensor.get_attribute("soc_in_mwh") is not None
-        ):
-            soc_at_start = sensor.get_attribute("soc_in_mwh")
-        else:
-            soc_at_start = 0
-
-    if soc_targets is None:
-        soc_targets = pd.Series(
-            np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
-        )
-
-    data_source_name = "Seita"
-
     # Choose which algorithm to use
     if "custom-scheduler" in sensor.attributes:
         scheduler_specs = sensor.attributes.get("custom-scheduler")
@@ -171,11 +154,7 @@ def make_schedule(
         start,
         end,
         resolution,
-        soc_at_start,
-        soc_targets,
-        soc_min,
-        soc_max,
-        roundtrip_efficiency,
+        storage_specs=storage_specs,
         consumption_price_sensor=consumption_price_sensor,
         production_price_sensor=production_price_sensor,
         inflexible_device_sensors=inflexible_device_sensors,
diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
index f1f4bd5ea..272a79f14 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
@@ -47,8 +47,7 @@ def test_scheduling_a_charging_station(
         end,
         belief_time=start,
         resolution=resolution,
-        soc_at_start=soc_at_start,
-        soc_targets=soc_targets,
+        storage_specs=dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
     )
 
     print("Job: %s" % job.id)

From 3bed7c0c6d2968515d522fb850cde2c93dd79079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sat, 1 Oct 2022 23:08:58 +0200
Subject: [PATCH 02/44] add changelog entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 documentation/changelog.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index dd56a80ae..83fe30243 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -8,7 +8,7 @@ v0.12.0 | October XX, 2022
 New features
 -------------
 
-* Hit the replay button to replay what happened, available on the sensor and asset pages [see `PR #463 <http://www.github.com/FlexMeasures/flexmeasures/pull/463>`_]
+* Hit the replay button to visually replay what happened, available on the sensor and asset pages [see `PR #463 <http://www.github.com/FlexMeasures/flexmeasures/pull/463>`_]
 * Ability to provide your own custom scheduling function [see `PR #505 <http://www.github.com/FlexMeasures/flexmeasures/pull/505>`_]
 * Visually distinguish forecasts/schedules (dashed lines) from measurements (solid lines), and expand the tooltip with timing info regarding the forecast/schedule horizon or measurement lag [see `PR #503 <http://www.github.com/FlexMeasures/flexmeasures/pull/503>`_]
 * The asset page also allows to show sensor data from other assets that belong to the same account [see `PR #500 <http://www.github.com/FlexMeasures/flexmeasures/pull/500>`_]
@@ -23,6 +23,7 @@ Infrastructure / Support
 
 * Reduce size of Docker image (from 2GB to 1.4GB) [see `PR #512 <http://www.github.com/FlexMeasures/flexmeasures/pull/512>`_]
 * Remove bokeh dependency and obsolete UI views [see `PR #476 <http://www.github.com/FlexMeasures/flexmeasures/pull/476>`_]
+* Improve documentation and code w.r.t. storage flexibility mnodeling [see `PR #511 <http://www.github.com/FlexMeasures/flexmeasures/pull/511>`_]
 
 
 v0.11.2 | September 6, 2022

From 1fba29e8230bb7c1fa62c74b52a0ace45bd51519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sat, 1 Oct 2022 23:11:59 +0200
Subject: [PATCH 03/44] make tests work, include updating older API versions,
 make prefer_charging_sooner part of storage specs & an optional parameter in
 API v3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/api/v1_2/implementations.py      | 16 +++--
 flexmeasures/api/v1_3/implementations.py      | 12 ++--
 flexmeasures/api/v3_0/sensors.py              |  6 ++
 flexmeasures/data/models/planning/battery.py  |  4 +-
 .../data/models/planning/charging_station.py  |  2 +-
 .../data/models/planning/tests/test_solver.py | 60 +++++++++++++++----
 flexmeasures/data/models/planning/utils.py    | 13 ++--
 7 files changed, 83 insertions(+), 30 deletions(-)

diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py
index 5f5a9fef3..647586672 100644
--- a/flexmeasures/api/v1_2/implementations.py
+++ b/flexmeasures/api/v1_2/implementations.py
@@ -39,6 +39,7 @@
 )
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.services.resources import has_assets, can_access_asset
+from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.utils.time_utils import duration_isoformat
 
 
@@ -93,17 +94,20 @@ def get_device_message_response(generic_asset_name_groups, duration):
             start = datetime.fromisoformat(
                 sensor.generic_asset.get_attribute("soc_datetime")
             )
+            end = start + planning_horizon
             resolution = sensor.event_resolution
 
             # Schedule the asset
+            storage_specs = dict(
+                soc_at_start=sensor.generic_asset.get_attribute("soc_in_mwh"),
+                prefer_charging_sooner=False,
+            )
+            storage_specs = ensure_storage_specs(
+                storage_specs, sensor_id, start, end, resolution
+            )
             try:
                 schedule = schedule_battery(
-                    sensor,
-                    start,
-                    start + planning_horizon,
-                    resolution,
-                    soc_at_start=sensor.generic_asset.get_attribute("soc_in_mwh"),
-                    prefer_charging_sooner=False,
+                    sensor, start, end, resolution, storage_specs=storage_specs
                 )
             except UnknownPricesException:
                 return unknown_prices()
diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index 8b2a5b857..1aa0da4b6 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -364,11 +364,13 @@ def post_udi_event_response(unit: str, prior: datetime):
         end_of_schedule,
         resolution=resolution,
         belief_time=prior,  # server time if no prior time was sent
-        soc_at_start=value,
-        soc_targets=soc_targets,
-        soc_min=soc_min,
-        soc_max=soc_max,
-        roundtrip_efficiency=roundtrip_efficiency,
+        storage_specs=dict(
+            soc_at_start=value,
+            soc_targets=soc_targets,
+            soc_min=soc_min,
+            soc_max=soc_max,
+            roundtrip_efficiency=roundtrip_efficiency,
+        ),
         job_id=form.get("event"),
         enqueue=True,
     )
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index 0cf7076ee..de1f80f9a 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -220,6 +220,9 @@ def get_data(self, response: dict):
                 ),
             ),  # todo: allow unit to be set per field, using QuantityField("%", validate=validate.Range(min=0, max=1))
             "targets": fields.List(fields.Nested(TargetSchema), data_key="soc-targets"),
+            "prefer_charging_sooner": fields.Bool(
+                data_key="prefer-charging-sooner", required=False
+            ),
             # todo: add a duration parameter, instead of falling back to FLEXMEASURES_PLANNING_HORIZON
             "consumption_price_sensor": SensorIdField(
                 data_key="consumption-price-sensor", required=False
@@ -241,6 +244,7 @@ def trigger_schedule(  # noqa: C901
         unit: str,
         prior: datetime,
         roundtrip_efficiency: Optional[ur.Quantity] = None,
+        prefer_charging_sooner: Optional[bool] = True,
         consumption_price_sensor: Optional[Sensor] = None,
         production_price_sensor: Optional[Sensor] = None,
         inflexible_device_sensors: Optional[List[Sensor]] = None,
@@ -274,6 +278,7 @@ def trigger_schedule(  # noqa: C901
             - soc-max (defaults to max soc target)
             - soc-targets (defaults to NaN values)
             - roundtrip-efficiency (defaults to 100%)
+            - prefer-charging-sooner (defaults to True)
 
         2) Heat pump sensors are work in progress.
 
@@ -442,6 +447,7 @@ def trigger_schedule(  # noqa: C901
                 soc_min=soc_min,
                 soc_max=soc_max,
                 roundtrip_efficiency=roundtrip_efficiency,
+                prefer_charging_sooner=prefer_charging_sooner,
             ),
             consumption_price_sensor=consumption_price_sensor,
             production_price_sensor=production_price_sensor,
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index c654f15fd..035b6f873 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -21,7 +21,6 @@ def schedule_battery(
     end: datetime,
     resolution: timedelta,
     storage_specs: dict,
-    prefer_charging_sooner: bool = True,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
     inflexible_device_sensors: Optional[List[Sensor]] = None,
@@ -38,6 +37,7 @@ def schedule_battery(
     soc_min = storage_specs.get("soc_min")
     soc_max = storage_specs.get("soc_max")
     roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+    prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
 
     # Check for required Sensor attributes
     sensor.check_required_attributes(
@@ -66,7 +66,6 @@ def schedule_battery(
         allow_trimmed_query_window=True,
     )
 
-    soc_targets = soc_targets.tz_convert("UTC")
     start = pd.Timestamp(start).tz_convert("UTC")
     end = pd.Timestamp(end).tz_convert("UTC")
 
@@ -114,6 +113,7 @@ def schedule_battery(
             sensor=inflexible_sensor,
         )
     if soc_targets is not None:
+        soc_targets = soc_targets.tz_convert("UTC")
         device_constraints[0]["equals"] = soc_targets.shift(
             -1, freq=resolution
         ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 25461a757..f326605f0 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -21,7 +21,6 @@ def schedule_charging_station(
     end: datetime,
     resolution: timedelta,
     storage_specs: dict,
-    prefer_charging_sooner: bool = True,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
     inflexible_device_sensors: Optional[List[Sensor]] = None,
@@ -39,6 +38,7 @@ def schedule_charging_station(
     soc_min = storage_specs.get("soc_min")
     soc_max = storage_specs.get("soc_max")
     roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+    prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
 
     # Check for required Sensor attributes
     sensor.check_required_attributes([("capacity_in_mw", (float, int))])
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 15396e66b..a53126424 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -8,6 +8,7 @@
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.models.planning.battery import schedule_battery
 from flexmeasures.data.models.planning.charging_station import schedule_charging_station
+from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.utils.calculations import integrate_time_series
 
 
@@ -26,12 +27,15 @@ def test_battery_solver_day_1(
     end = tz.localize(datetime(2015, 1, 2))
     resolution = timedelta(minutes=15)
     soc_at_start = battery.get_attribute("soc_in_mwh")
+    storage_specs = ensure_storage_specs(
+        dict(soc_at_start=soc_at_start), battery.id, start, end, resolution
+    )
     schedule = schedule_battery(
         battery,
         start,
         end,
         resolution,
-        soc_at_start,
+        storage_specs=storage_specs,
         inflexible_device_sensors=add_inflexible_device_forecasts.keys()
         if use_inflexible_device
         else None,
@@ -80,15 +84,24 @@ def test_battery_solver_day_2(add_battery_assets, roundtrip_efficiency: float):
     soc_at_start = battery.get_attribute("soc_in_mwh")
     soc_min = 0.5
     soc_max = 4.5
+    storage_specs = ensure_storage_specs(
+        dict(
+            soc_at_start=soc_at_start,
+            soc_min=soc_min,
+            soc_max=soc_max,
+            roundtrip_efficiency=roundtrip_efficiency,
+        ),
+        battery.id,
+        start,
+        end,
+        resolution,
+    )
     schedule = schedule_battery(
         battery,
         start,
         end,
         resolution,
-        soc_at_start,
-        soc_min=soc_min,
-        soc_max=soc_max,
-        roundtrip_efficiency=roundtrip_efficiency,
+        storage_specs=storage_specs,
     )
     soc_schedule = integrate_time_series(schedule, soc_at_start, decimal_precision=6)
 
@@ -157,8 +170,15 @@ def test_charging_station_solver_day_2(target_soc, charging_station_name):
         np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
     )
     soc_targets.loc[target_soc_datetime] = target_soc
+    storage_specs = ensure_storage_specs(
+        dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
+        charging_station.id,
+        start,
+        end,
+        resolution,
+    )
     consumption_schedule = schedule_charging_station(
-        charging_station, start, end, resolution, soc_at_start, soc_targets
+        charging_station, start, end, resolution, storage_specs=storage_specs
     )
     soc_schedule = integrate_time_series(
         consumption_schedule, soc_at_start, decimal_precision=6
@@ -214,8 +234,19 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
         np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
     )
     soc_targets.loc[target_soc_datetime] = target_soc
+    storage_specs = ensure_storage_specs(
+        dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
+        charging_station.id,
+        start,
+        end,
+        resolution,
+    )
     consumption_schedule = schedule_charging_station(
-        charging_station, start, end, resolution, soc_at_start, soc_targets
+        charging_station,
+        start,
+        end,
+        resolution,
+        storage_specs=storage_specs,
     )
     soc_schedule = integrate_time_series(
         consumption_schedule, soc_at_start, decimal_precision=6
@@ -263,14 +294,23 @@ def test_building_solver_day_2(
     soc_at_start = 2.5
     soc_min = 0.5
     soc_max = 4.5
+    storage_specs = ensure_storage_specs(
+        dict(
+            soc_at_start=soc_at_start,
+            soc_min=soc_min,
+            soc_max=soc_max,
+        ),
+        battery.id,
+        start,
+        end,
+        resolution,
+    )
     schedule = schedule_battery(
         battery,
         start,
         end,
         resolution,
-        soc_at_start,
-        soc_min=soc_min,
-        soc_max=soc_max,
+        storage_specs=storage_specs,
         inflexible_device_sensors=inflexible_devices.values(),
     )
     soc_schedule = integrate_time_series(schedule, soc_at_start, decimal_precision=6)
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 5ab769174..b221a7b50 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -44,7 +44,7 @@ def initialize_index(
 
 def ensure_storage_specs(
     specs: Optional[dict],
-    sensor_id: Sensor,
+    sensor_id: int,
     start_of_schedule: datetime,
     end_of_schedule: datetime,
     resolution: timedelta,
@@ -58,6 +58,7 @@ def ensure_storage_specs(
     - soc_max
     - soc_targets
     - roundtrip_efficiency
+    - prefer_charging_sooner
     """
     if specs is None:
         specs = {}
@@ -74,7 +75,7 @@ def ensure_sensor_is_set(sensor) -> Sensor:
     # Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
     # Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
     # and without soc targets and limits the starting soc doesn't matter).
-    if "soc_at_start" not in specs:
+    if "soc_at_start" not in specs or specs["soc_at_start"] is None:
         sensor = ensure_sensor_is_set(sensor)
         if (
             start_of_schedule == sensor.get_attribute("soc_datetime")
@@ -85,7 +86,7 @@ def ensure_sensor_is_set(sensor) -> Sensor:
             specs["soc_at_start"] = 0
 
     # init targets
-    if "soc_targets" not in specs:
+    if "soc_targets" not in specs or specs["soc_targets"] is None:
         specs["soc_targets"] = pd.Series(
             np.nan,
             index=pd.date_range(
@@ -98,11 +99,11 @@ def ensure_sensor_is_set(sensor) -> Sensor:
     ]
 
     # Check for min and max SOC, or get default from sensor
-    if "soc_min" not in specs:
+    if "soc_min" not in specs or specs["soc_min"] is None:
         sensor = ensure_sensor_is_set(sensor)
         # Can't drain the EV battery by more than it contains
         specs["soc_min"] = sensor.get_attribute("min_soc_in_mwh", 0)
-    if "soc_max" not in specs:
+    if "soc_max" not in specs or specs["soc_max"] is None:
         sensor = ensure_sensor_is_set(sensor)
         # Lacking information about the battery's nominal capacity, we use the highest target value as the maximum state of charge
         specs["soc_max"] = sensor.get_attribute(
@@ -110,7 +111,7 @@ def ensure_sensor_is_set(sensor) -> Sensor:
         )
 
     # Check for round-trip efficiency
-    if "roundtrip_efficiency" not in specs:
+    if "roundtrip_efficiency" not in specs or specs["roundtrip_efficiency"] is None:
         sensor = ensure_sensor_is_set(sensor)
         # Get default from sensor, or use 100% otherwise
         specs["roundtrip_efficiency"] = sensor.get_attribute("roundtrip_efficiency", 1)

From 0e9259d44ca2cab79ac2a5550cda3b2c921ab31d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sun, 2 Oct 2022 00:28:13 +0200
Subject: [PATCH 04/44] use storage_specs in CLI command, as well
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/cli/data_add.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py
index f4e4131f2..08a7f345f 100755
--- a/flexmeasures/cli/data_add.py
+++ b/flexmeasures/cli/data_add.py
@@ -958,11 +958,13 @@ def create_schedule(
             end_of_schedule=end,
             belief_time=server_now(),
             resolution=power_sensor.event_resolution,
-            soc_at_start=soc_at_start,
-            soc_targets=soc_targets,
-            soc_min=soc_min,
-            soc_max=soc_max,
-            roundtrip_efficiency=roundtrip_efficiency,
+            storage_specs=dict(
+                soc_at_start=soc_at_start,
+                soc_targets=soc_targets,
+                soc_min=soc_min,
+                soc_max=soc_max,
+                roundtrip_efficiency=roundtrip_efficiency,
+            ),
             consumption_price_sensor=consumption_price_sensor,
             production_price_sensor=production_price_sensor,
         )

From c2b278751f1b9ae1c10e60d8920b31222b32a660 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sun, 2 Oct 2022 12:27:42 +0200
Subject: [PATCH 05/44] remove default resolution of 15M, for now pass in what
 you want
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 2 +-
 flexmeasures/data/services/scheduling.py   | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index b221a7b50..783033a6d 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -85,7 +85,7 @@ def ensure_sensor_is_set(sensor) -> Sensor:
         else:
             specs["soc_at_start"] = 0
 
-    # init targets
+    # init default targets
     if "soc_targets" not in specs or specs["soc_targets"] is None:
         specs["soc_targets"] = pd.Series(
             np.nan,
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 00e1b57e2..d5319cf9b 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -26,17 +26,13 @@
    This might re-enqueue the job or try a different model (which creates a new job).
 """
 
-DEFAULT_RESOLUTION = timedelta(
-    minutes=15
-)  # make_schedule can also fallback to sensor.event_resolution, trigger_schedule also uses that
-
 
 def create_scheduling_job(
     sensor_id: int,
     start_of_schedule: datetime,
     end_of_schedule: datetime,
     belief_time: datetime,
-    resolution: timedelta = DEFAULT_RESOLUTION,
+    resolution: timedelta,
     consumption_price_sensor: Optional[Sensor] = None,
     production_price_sensor: Optional[Sensor] = None,
     inflexible_device_sensors: Optional[List[Sensor]] = None,

From c0400ae6655004efdce83cc967439a5438004732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Fri, 28 Oct 2022 13:45:09 +0200
Subject: [PATCH 06/44] various review comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 documentation/plugin/customisation.rst     |  8 ++++--
 flexmeasures/api/v3_0/sensors.py           | 13 ++++-----
 flexmeasures/data/models/planning/utils.py | 31 +++++++++++++++++-----
 flexmeasures/data/services/scheduling.py   |  4 +--
 flexmeasures/data/tests/dummy_scheduler.py |  7 +++--
 5 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst
index bbe51fafe..b694f2b6c 100644
--- a/documentation/plugin/customisation.rst
+++ b/documentation/plugin/customisation.rst
@@ -33,9 +33,13 @@ The following minimal example gives you an idea of the inputs and outputs:
         *args,
         **kwargs
     ):
-        """Just a dummy scheduler, advising to do nothing"""
+        """
+        Just a dummy scheduler that always plans to consume at maximum capacity.
+        (Schedulers return positive values for consumption, and negative values for production)
+        """
         return pd.Series(
-            0, index=pd.date_range(start, end, freq=resolution, closed="left")
+            sensor.get_attribute("capacity_in_mw"),
+            index=pd.date_range(start, end, freq=resolution, closed="left"),
         )
 
 
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index de1f80f9a..aedf09742 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -256,20 +256,21 @@ def trigger_schedule(  # noqa: C901
         .. :quickref: Schedule; Trigger scheduling job
 
         Trigger FlexMeasures to create a schedule for this sensor.
-        The assumption is that this sensor is the energy sensor on a flexible asset.
+        The assumption is that this sensor is the power sensor on a flexible asset.
 
         In this request, you can describe:
 
         - the schedule (start, unit, prior)
         - the flexibility model for the sensor (see below, only storage models are supported at the moment)
-        - the EMS the sensor operates in (inflexible device sensors, sensors which put a price on consumption and/or production)
+        - the EMS the sensor operates in (inflexible device sensors, and sensors that put a price on consumption and/or production)
 
-        Note: This endpoint does not support an EMS with multiple flexible sensors. This will happen in another endpoint.
-              See https://github.com/FlexMeasures/flexmeasures/issues/485
+        Note: This endpoint does not support to schedule an EMS with multiple flexible sensors at once. This will happen in another endpoint.
+              See https://github.com/FlexMeasures/flexmeasures/issues/485. Until then, it is possible to call this endpoint for one flexible endpoint at a time
+              (considering already scheduled sensors as inflexible).
 
         Flexibility models apply to the sensor's asset type:
 
-        1) For storage sensors (e.g. battery, charging stations), the schedule deals with the state of charge (SOC).
+        1) For storage sensors (e.g. battery, charge points), the schedule deals with the state of charge (SOC).
            The possible flexibility parameters are:
 
             - soc-at-start (defaults to 0)
@@ -365,7 +366,7 @@ def trigger_schedule(  # noqa: C901
         if unit == "kWh":
             start_value = start_value / 1000.0
 
-        # Convert round-trip efficiency to dimensionless (to the [0,1] range)
+        # Convert round-trip efficiency to dimensionless (to the (0,1] range)
         if roundtrip_efficiency is not None:
             roundtrip_efficiency = roundtrip_efficiency.to(
                 ur.Quantity("dimensionless")
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 783033a6d..56fcda75a 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -17,9 +17,15 @@
 
 
 def initialize_df(
-    columns: List[str], start: datetime, end: datetime, resolution: timedelta
+    columns: List[str],
+    start: datetime,
+    end: datetime,
+    resolution: timedelta,
+    inclusive: str,
 ) -> pd.DataFrame:
-    df = pd.DataFrame(index=initialize_index(start, end, resolution), columns=columns)
+    df = pd.DataFrame(
+        index=initialize_index(start, end, resolution, inclusive), columns=columns
+    )
     return df
 
 
@@ -28,16 +34,25 @@ def initialize_series(
     start: datetime,
     end: datetime,
     resolution: timedelta,
+    inclusive: str,
 ) -> pd.Series:
-    s = pd.Series(index=initialize_index(start, end, resolution), data=data)
+    s = pd.Series(index=initialize_index(start, end, resolution, inclusive), data=data)
     return s
 
 
 def initialize_index(
-    start: Union[date, datetime], end: Union[date, datetime], resolution: timedelta
+    start: Union[date, datetime],
+    end: Union[date, datetime],
+    resolution: timedelta,
+    inclusive: str = "left",
 ) -> pd.DatetimeIndex:
+    assert inclusive == "left" or inclusive == "right"
     i = pd.date_range(
-        start=start, end=end, freq=to_offset(resolution), closed="left", name="datetime"
+        start=start,
+        end=end,
+        freq=to_offset(resolution),
+        closed=inclusive,
+        name="datetime",
     )
     return i
 
@@ -101,7 +116,7 @@ def ensure_sensor_is_set(sensor) -> Sensor:
     # Check for min and max SOC, or get default from sensor
     if "soc_min" not in specs or specs["soc_min"] is None:
         sensor = ensure_sensor_is_set(sensor)
-        # Can't drain the EV battery by more than it contains
+        # Can't drain the storage by more than it contains
         specs["soc_min"] = sensor.get_attribute("min_soc_in_mwh", 0)
     if "soc_max" not in specs or specs["soc_max"] is None:
         sensor = ensure_sensor_is_set(sensor)
@@ -234,6 +249,10 @@ def get_power_values(
         raise UnknownForecastException(
             f"Forecasts unknown for planning window. (sensor {sensor.id})"
         )
+    if sensor.get_attribute(
+        "consumption_is_positive", False
+    ):  # FlexMeasures default is to store consumption as negative power values
+        return df.values
     return -df.values
 
 
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index d5319cf9b..59ff663e3 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -109,7 +109,7 @@ def make_schedule(
 
     - Choose which scheduling function can be used
     - Compute schedule
-    - Turn schedukled values into beliefs and save them to db
+    - Turn scheduled values into beliefs and save them to db
     """
     # https://docs.sqlalchemy.org/en/13/faq/connections.html#how-do-i-use-engines-connections-sessions-with-python-multiprocessing-or-os-fork
     db.engine.dispose()
@@ -202,7 +202,7 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]:
     assert "function" in scheduler_specs, "scheduler specs have no 'function'"
 
     source_name = scheduler_specs.get(
-        "source", f"Custom scheduler - {scheduler_specs['function']}"
+        "source", f"custom scheduler - {scheduler_specs['function']}"
     )
     scheduler_name = scheduler_specs["function"]
 
diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py
index cf4f0ec89..207c98e52 100644
--- a/flexmeasures/data/tests/dummy_scheduler.py
+++ b/flexmeasures/data/tests/dummy_scheduler.py
@@ -12,10 +12,13 @@ def compute_a_schedule(
     *args,
     **kwargs
 ):
-    """Just a dummy scheduler."""
+    """
+    Just a dummy scheduler that always plans to consume at maximum capacity.
+    (Schedulers return positive values for consumption, and negative values for production)
+    """
     return initialize_series(  # simply creates a Pandas Series repeating one value
         data=sensor.get_attribute("capacity_in_mw"),
         start=start,
         end=end,
         resolution=resolution,
-    )
+    ) 

From 4718a8aa4fe627fb476bbb9c3a2b4a99fd6e8a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sat, 29 Oct 2022 20:52:23 +0200
Subject: [PATCH 07/44] black
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/data/tests/dummy_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py
index 207c98e52..5260cdcb4 100644
--- a/flexmeasures/data/tests/dummy_scheduler.py
+++ b/flexmeasures/data/tests/dummy_scheduler.py
@@ -21,4 +21,4 @@ def compute_a_schedule(
         start=start,
         end=end,
         resolution=resolution,
-    ) 
+    )

From 2eef27d9d920c3b851f889e20dce700a4c4be883 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Sat, 29 Oct 2022 21:40:39 +0200
Subject: [PATCH 08/44] fix tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/data/models/planning/utils.py      | 4 ++--
 flexmeasures/data/tests/test_scheduling_jobs.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 56fcda75a..393c3ca79 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -21,7 +21,7 @@ def initialize_df(
     start: datetime,
     end: datetime,
     resolution: timedelta,
-    inclusive: str,
+    inclusive: str = "left",
 ) -> pd.DataFrame:
     df = pd.DataFrame(
         index=initialize_index(start, end, resolution, inclusive), columns=columns
@@ -34,7 +34,7 @@ def initialize_series(
     start: datetime,
     end: datetime,
     resolution: timedelta,
-    inclusive: str,
+    inclusive: str = "left",
 ) -> pd.Series:
     s = pd.Series(index=initialize_index(start, end, resolution, inclusive), data=data)
     return s
diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py
index a0111a445..25897567b 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs.py
@@ -82,7 +82,7 @@ def test_loading_custom_scheduler(is_path: bool):
     custom_scheduler, data_source = load_custom_scheduler(scheduler_specs)
     assert data_source == "Test Source"
     assert custom_scheduler.__name__ == "compute_a_schedule"
-    assert custom_scheduler.__doc__ == "Just a dummy scheduler."
+    assert "Just a dummy scheduler" in custom_scheduler.__doc__
 
 
 @pytest.mark.parametrize("is_path", [False, True])

From 1f68659712fe141596321ab52dee01edcce34d57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Mon, 31 Oct 2022 06:44:46 +0100
Subject: [PATCH 09/44] always load sensor when checking storage specs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 393c3ca79..60de696e9 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -78,12 +78,7 @@ def ensure_storage_specs(
     if specs is None:
         specs = {}
 
-    sensor: Optional[Sensor] = None
-
-    def ensure_sensor_is_set(sensor) -> Sensor:
-        if sensor is None:
-            sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
-        return sensor
+    sensor: Optional[Sensor] = Sensor.query.filter_by(id=sensor_id).one_or_none()
 
     # Check state of charge
     # Preferably, a starting soc is given.
@@ -91,7 +86,6 @@ def ensure_sensor_is_set(sensor) -> Sensor:
     # Otherwise, we set the starting soc to 0 (some assets don't use the concept of a state of charge,
     # and without soc targets and limits the starting soc doesn't matter).
     if "soc_at_start" not in specs or specs["soc_at_start"] is None:
-        sensor = ensure_sensor_is_set(sensor)
         if (
             start_of_schedule == sensor.get_attribute("soc_datetime")
             and sensor.get_attribute("soc_in_mwh") is not None
@@ -115,11 +109,9 @@ def ensure_sensor_is_set(sensor) -> Sensor:
 
     # Check for min and max SOC, or get default from sensor
     if "soc_min" not in specs or specs["soc_min"] is None:
-        sensor = ensure_sensor_is_set(sensor)
         # Can't drain the storage by more than it contains
         specs["soc_min"] = sensor.get_attribute("min_soc_in_mwh", 0)
     if "soc_max" not in specs or specs["soc_max"] is None:
-        sensor = ensure_sensor_is_set(sensor)
         # Lacking information about the battery's nominal capacity, we use the highest target value as the maximum state of charge
         specs["soc_max"] = sensor.get_attribute(
             "max_soc_in_mwh", max(specs["soc_targets"].values)
@@ -127,7 +119,6 @@ def ensure_sensor_is_set(sensor) -> Sensor:
 
     # Check for round-trip efficiency
     if "roundtrip_efficiency" not in specs or specs["roundtrip_efficiency"] is None:
-        sensor = ensure_sensor_is_set(sensor)
         # Get default from sensor, or use 100% otherwise
         specs["roundtrip_efficiency"] = sensor.get_attribute("roundtrip_efficiency", 1)
     if specs["roundtrip_efficiency"] <= 0 or specs["roundtrip_efficiency"] > 1:

From 2beb928f52f2505dca5d49651d42193c7d2199f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Mon, 31 Oct 2022 19:16:33 +0100
Subject: [PATCH 10/44] begin to handle source model and version during
 scheduling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 documentation/plugin/customisation.rst        |  9 ++--
 flexmeasures/api/v1_3/implementations.py      | 21 ++++++----
 flexmeasures/api/v1_3/tests/test_api_v1_3.py  |  6 ++-
 flexmeasures/api/v3_0/sensors.py              | 16 +++++---
 .../api/v3_0/tests/test_sensor_schedules.py   |  6 ++-
 flexmeasures/data/models/planning/battery.py  |  4 ++
 flexmeasures/data/services/scheduling.py      | 41 +++++++++++--------
 flexmeasures/data/tests/dummy_scheduler.py    |  4 ++
 .../data/tests/test_scheduling_jobs.py        | 12 +++---
 9 files changed, 78 insertions(+), 41 deletions(-)

diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst
index b694f2b6c..52fe57397 100644
--- a/documentation/plugin/customisation.rst
+++ b/documentation/plugin/customisation.rst
@@ -16,7 +16,7 @@ but in the background your custom scheduling algorithm is being used.
 Let's walk through an example!
 
 First, we need to write a function which accepts arguments just like the in-built schedulers (their code is `here <https://github.com/FlexMeasures/flexmeasures/tree/main/flexmeasures/data/models/planning>`_).
-The following minimal example gives you an idea of the inputs and outputs:
+The following minimal example gives you an idea of some meta information you can add for labeling your data, as well as the inputs and outputs of such a scheduling function:
 
 .. code-block:: python
 
@@ -25,6 +25,10 @@ The following minimal example gives you an idea of the inputs and outputs:
     from pandas.tseries.frequencies import to_offset
     from flexmeasures.data.models.time_series import Sensor
 
+
+    __author__ = "My Company"
+    __version__ = "v2"
+
     def compute_a_schedule(
         sensor: Sensor,
         start: datetime,
@@ -44,7 +48,7 @@ The following minimal example gives you an idea of the inputs and outputs:
 
 
 .. note:: It's possible to add arguments that describe the asset flexibility and the EMS context in more detail. For example,
-          for storage assets we support various state-of-charge parameters. For now, the existing schedulers are the best documentation.
+          for storage assets we support various state-of-charge parameters. For now, the existing in-built schedulers are the best documentation.
 
 
 Finally, make your scheduler be the one that FlexMeasures will use for certain sensors:
@@ -57,7 +61,6 @@ Finally, make your scheduler be the one that FlexMeasures will use for certain s
     scheduler_specs = {
         "module": "flexmeasures.data.tests.dummy_scheduler",  # or a file path, see note below
         "function": "compute_a_schedule",
-        "source": "My Company"
     }
     
     my_sensor = Sensor.query.filter(Sensor.name == "My power sensor on a flexible asset").one_or_none()
diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index 1aa0da4b6..2193c919e 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -99,6 +99,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
             if event_type not in ("soc", "soc-with-targets"):
                 return unrecognized_event_type(event_type)
             connection = current_app.queues["scheduling"].connection
+            job = None
             try:  # First try the scheduling queue
                 job = Job.fetch(event, connection=connection)
             except NoSuchJobError:  # Then try the most recent event_id (stored as a generic asset attribute)
@@ -144,14 +145,20 @@ def get_device_message_response(generic_asset_name_groups, duration):
                     return unknown_schedule("Scheduling job has an unknown status.")
                 schedule_start = job.kwargs["start"]
 
-            schedule_data_source_name = "Seita"
-            scheduler_source = DataSource.query.filter_by(
-                name=schedule_data_source_name, type="scheduling script"
-            ).one_or_none()
+            data_source_info = None
+            if job:
+                data_source_info = job.meta.get("data_source_info")
+            if data_source_info is None:
+                data_source_info = dict(
+                    name="Seita"
+                )  # TODO: change to raise later - all scheduling jobs now get full info
+            scheduler_sources = DataSource.query.filter_by(
+                type="scheduling script",
+                **data_source_info,
+            ).all()  # Might be more than one, e.g. per user
             if scheduler_source is None:
-                return unknown_schedule(
-                    message + f'no data is known from "{schedule_data_source_name}".'
-                )
+                s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
+                return unknown_schedule(message + f"no data is known from [{s_info}].")
 
             power_values = sensor.search_beliefs(
                 event_starts_after=schedule_start,
diff --git a/flexmeasures/api/v1_3/tests/test_api_v1_3.py b/flexmeasures/api/v1_3/tests/test_api_v1_3.py
index 54429774a..8084405af 100644
--- a/flexmeasures/api/v1_3/tests/test_api_v1_3.py
+++ b/flexmeasures/api/v1_3/tests/test_api_v1_3.py
@@ -88,9 +88,10 @@ def test_post_udi_event_and_get_device_message(
     )
 
     # check results are in the database
-    resolution = timedelta(minutes=15)
+    job.refresh()  # catch meta info that was added on this very instance
+    data_source_info = job.meta.get("data_source_info")
     scheduler_source = DataSource.query.filter_by(
-        name="Seita", type="scheduling script"
+        type="scheduling script", **data_source_info
     ).one_or_none()
     assert (
         scheduler_source is not None
@@ -100,6 +101,7 @@ def test_post_udi_event_and_get_device_message(
         .filter(TimedBelief.source_id == scheduler_source.id)
         .all()
     )
+    resolution = timedelta(minutes=15)
     consumption_schedule = pd.Series(
         [-v.event_value for v in power_values],
         index=pd.DatetimeIndex([v.event_start for v in power_values], freq=resolution),
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index aedf09742..81de11695 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -552,15 +552,19 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
             return unknown_schedule("Scheduling job has an unknown status.")
         schedule_start = job.kwargs["start"]
 
-        schedule_data_source_name = "Seita"
-        if "data_source_name" in job.meta:
-            schedule_data_source_name = job.meta["data_source_name"]
+        data_source_info = job.meta.get("data_source_info")
+        if data_source_info is None:
+            data_source_info = dict(
+                name="Seita"
+            )  # TODO: change to raise later - all scheduling jobs now get full info
         scheduler_source = DataSource.query.filter_by(
-            name=schedule_data_source_name, type="scheduling script"
-        ).one_or_none()
+            type="scheduling script",
+            **data_source_info,
+        ).one_or_none()  # this assumes full info, otherwise there can be more than one
         if scheduler_source is None:
+            s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
             return unknown_schedule(
-                error_message + f'no data is known from "{schedule_data_source_name}".'
+                error_message + f"no data is known from [{s_info}]."
             )
 
         power_values = sensor.search_beliefs(
diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
index 9c9ac912a..5a94668ab 100644
--- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
+++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
@@ -66,9 +66,10 @@ def test_trigger_and_get_schedule(
     )
 
     # check results are in the database
-    resolution = timedelta(minutes=15)
+    job.refresh()  # catch meta info that was added on this very instance
+    data_source_info = job.meta.get("data_source_info")
     scheduler_source = DataSource.query.filter_by(
-        name="Seita", type="scheduling script"
+        type="scheduling script", **data_source_info
     ).one_or_none()
     assert (
         scheduler_source is not None
@@ -78,6 +79,7 @@ def test_trigger_and_get_schedule(
         .filter(TimedBelief.source_id == scheduler_source.id)
         .all()
     )
+    resolution = timedelta(minutes=15)
     consumption_schedule = pd.Series(
         [-v.event_value for v in power_values],
         index=pd.DatetimeIndex([v.event_start for v in power_values], freq=resolution),
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 035b6f873..285fee1d1 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -15,6 +15,10 @@
 )
 
 
+# __version__ = "1"
+# __author__ = "Seita"
+
+
 def schedule_battery(
     sensor: Sensor,
     start: datetime,
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 59ff663e3..cd41959f5 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -115,7 +115,9 @@ def make_schedule(
     db.engine.dispose()
 
     sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
-    data_source_name = "Seita"
+    data_source_info = dict(
+        name="Seita", model="Unknown", version="1"
+    )  # will be overwritten by scheduler choice below
 
     rq_job = get_current_job()
     if rq_job:
@@ -127,24 +129,27 @@ def make_schedule(
     # Choose which algorithm to use
     if "custom-scheduler" in sensor.attributes:
         scheduler_specs = sensor.attributes.get("custom-scheduler")
-        scheduler, data_source_name = load_custom_scheduler(scheduler_specs)
-        if rq_job:
-            rq_job.meta["data_source_name"] = data_source_name
-            rq_job.save_meta()
+        scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
     elif sensor.generic_asset.generic_asset_type.name == "battery":
         scheduler = schedule_battery
+        data_source_info["model"] = "schedule_battery"
     elif sensor.generic_asset.generic_asset_type.name in (
         "one-way_evse",
         "two-way_evse",
     ):
         scheduler = schedule_charging_station
-
+        data_source_info["model"] = "schedule_charging_station"
     else:
         raise ValueError(
             "Scheduling is not (yet) supported for asset type %s."
             % sensor.generic_asset.generic_asset_type
         )
 
+    # saving info on the job, so the API for a job can look the data up
+    if rq_job:
+        rq_job.meta["data_source_info"] = data_source_info
+        rq_job.save_meta()
+
     consumption_schedule = scheduler(
         sensor,
         start,
@@ -156,13 +161,15 @@ def make_schedule(
         inflexible_device_sensors=inflexible_device_sensors,
         belief_time=belief_time,
     )
+    if rq_job:
+        click.echo("Job %s made schedule." % rq_job.id)
 
     data_source = get_data_source(
-        data_source_name=data_source_name,
+        data_source_name=data_source_info["name"],
+        data_source_model=data_source_info["model"],
+        data_source_version=data_source_info["version"],
         data_source_type="scheduling script",
     )
-    if rq_job:
-        click.echo("Job %s made schedule." % rq_job.id)
 
     ts_value_schedule = [
         TimedBelief(
@@ -181,17 +188,16 @@ def make_schedule(
     return True
 
 
-def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]:
+def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, dict]:
     """
     Read in custom scheduling spec.
-    Attempt to load the Callable, also derive a data source name.
+    Attempt to load the Callable, also derive data source info.
 
     Example specs:
 
     {
         "module": "/path/to/module.py",  # or sthg importable, e.g. "package.module"
         "function": "name_of_function",
-        "source": "source name"
     }
 
     """
@@ -201,10 +207,8 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]:
     assert "module" in scheduler_specs, "scheduler specs have no 'module'."
     assert "function" in scheduler_specs, "scheduler specs have no 'function'"
 
-    source_name = scheduler_specs.get(
-        "source", f"custom scheduler - {scheduler_specs['function']}"
-    )
     scheduler_name = scheduler_specs["function"]
+    source_info = dict(model=scheduler_name, version="1", name="")  # default  # default
 
     # import module
     module_descr = scheduler_specs["module"]
@@ -233,7 +237,12 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, str]:
         module, scheduler_specs["function"]
     ), "Module at {module_descr} has no function {scheduler_specs['function']}"
 
-    return getattr(module, scheduler_specs["function"]), source_name
+    if hasattr(module, "__version__"):
+        source_info["version"] = str(module.__version__)
+    if hasattr(module, "__author__"):
+        source_info["name"] = str(module.__author__)
+
+    return getattr(module, scheduler_specs["function"]), source_info
 
 
 def handle_scheduling_exception(job, exc_type, exc_value, traceback):
diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py
index 5260cdcb4..b9193b440 100644
--- a/flexmeasures/data/tests/dummy_scheduler.py
+++ b/flexmeasures/data/tests/dummy_scheduler.py
@@ -4,6 +4,10 @@
 from flexmeasures.data.models.planning.utils import initialize_series
 
 
+__author__ = "Test Organization"
+__version__ = "v3"
+
+
 def compute_a_schedule(
     sensor: Sensor,
     start: datetime,
diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py
index 25897567b..da32b69e1 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs.py
@@ -61,7 +61,6 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data):
 scheduler_specs = {
     "module": None,  # use make_module_descr, see below
     "function": "compute_a_schedule",
-    "source": "Test Source",
 }
 
 
@@ -79,8 +78,10 @@ def test_loading_custom_scheduler(is_path: bool):
     Simply check if loading a custom scheduler works.
     """
     scheduler_specs["module"] = make_module_descr(is_path)
-    custom_scheduler, data_source = load_custom_scheduler(scheduler_specs)
-    assert data_source == "Test Source"
+    custom_scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
+    assert data_source_info["name"] == "Test Organization"
+    assert data_source_info["version"] == "v3"
+    assert data_source_info["model"] == "compute_a_schedule"
     assert custom_scheduler.__name__ == "compute_a_schedule"
     assert "Just a dummy scheduler" in custom_scheduler.__doc__
 
@@ -111,10 +112,11 @@ def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool):
     # make sure we saved the data source for later lookup
     redis_connection = app.queues["scheduling"].connection
     finished_job = Job.fetch(job.id, connection=redis_connection)
-    assert finished_job.meta["data_source_name"] == scheduler_specs["source"]
+    assert finished_job.meta["data_source_info"]["model"] == scheduler_specs["function"]
 
     scheduler_source = DataSource.query.filter_by(
-        name=finished_job.meta["data_source_name"], type="scheduling script"
+        type="scheduling script",
+        **finished_job.meta["data_source_info"],
     ).one_or_none()
     assert (
         scheduler_source is not None

From 378cabaf641e2497a9252e234254ba7d7e28750d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Tue, 1 Nov 2022 00:24:14 +0100
Subject: [PATCH 11/44] we can get multiple sources from our query (in the old
 setting, when we use name, but also in the new setting, unless we always
 include the user_id)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/api/v1_3/implementations.py | 4 ++--
 flexmeasures/api/v3_0/sensors.py         | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index 2193c919e..5db1dda98 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -156,14 +156,14 @@ def get_device_message_response(generic_asset_name_groups, duration):
                 type="scheduling script",
                 **data_source_info,
             ).all()  # Might be more than one, e.g. per user
-            if scheduler_source is None:
+            if len(scheduler_sources) == 0:
                 s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
                 return unknown_schedule(message + f"no data is known from [{s_info}].")
 
             power_values = sensor.search_beliefs(
                 event_starts_after=schedule_start,
                 event_ends_before=schedule_start + planning_horizon,
-                source=scheduler_source,
+                source=scheduler_sources[-1],
                 most_recent_beliefs_only=True,
                 one_deterministic_belief_per_event=True,
             )
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index 81de11695..ecf6a39ab 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -557,11 +557,11 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
             data_source_info = dict(
                 name="Seita"
             )  # TODO: change to raise later - all scheduling jobs now get full info
-        scheduler_source = DataSource.query.filter_by(
+        scheduler_sources = DataSource.query.filter_by(
             type="scheduling script",
             **data_source_info,
-        ).one_or_none()  # this assumes full info, otherwise there can be more than one
-        if scheduler_source is None:
+        ).all()  # there can be more than one, e.g. different users
+        if len(scheduler_sources) == 0:
             s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
             return unknown_schedule(
                 error_message + f"no data is known from [{s_info}]."
@@ -570,7 +570,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
         power_values = sensor.search_beliefs(
             event_starts_after=schedule_start,
             event_ends_before=schedule_start + planning_horizon,
-            source=scheduler_source,
+            source=scheduler_sources[-1],
             most_recent_beliefs_only=True,
             one_deterministic_belief_per_event=True,
         )

From 5b48baf04074fa1e6ef129c0b19a8222454b7041 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Tue, 1 Nov 2022 00:24:58 +0100
Subject: [PATCH 12/44] give our two in-built schedulers an official __author__
 and __version__
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/data/models/planning/battery.py          | 4 ++--
 flexmeasures/data/models/planning/charging_station.py | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 285fee1d1..ab6d6a676 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -15,8 +15,8 @@
 )
 
 
-# __version__ = "1"
-# __author__ = "Seita"
+__version__ = "1"
+__author__ = "Seita"
 
 
 def schedule_battery(
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index f326605f0..8ba57be47 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -14,6 +14,9 @@
     fallback_charging_policy,
 )
 
+__version__ = "1"
+__author__ = "Seita"
+
 
 def schedule_charging_station(
     sensor: Sensor,

From b664910d3b895c3aa849dfaafa152f7abc615564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Tue, 1 Nov 2022 00:29:10 +0100
Subject: [PATCH 13/44] review comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 documentation/changelog.rst                | 2 +-
 flexmeasures/api/v3_0/sensors.py           | 2 +-
 flexmeasures/data/models/planning/utils.py | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 83fe30243..3dd60757f 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -23,7 +23,7 @@ Infrastructure / Support
 
 * Reduce size of Docker image (from 2GB to 1.4GB) [see `PR #512 <http://www.github.com/FlexMeasures/flexmeasures/pull/512>`_]
 * Remove bokeh dependency and obsolete UI views [see `PR #476 <http://www.github.com/FlexMeasures/flexmeasures/pull/476>`_]
-* Improve documentation and code w.r.t. storage flexibility mnodeling [see `PR #511 <http://www.github.com/FlexMeasures/flexmeasures/pull/511>`_]
+* Improve documentation and code w.r.t. storage flexibility modelling [see `PR #511 <http://www.github.com/FlexMeasures/flexmeasures/pull/511>`_]
 
 
 v0.11.2 | September 6, 2022
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index ecf6a39ab..a167d7fbb 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -279,7 +279,7 @@ def trigger_schedule(  # noqa: C901
             - soc-max (defaults to max soc target)
             - soc-targets (defaults to NaN values)
             - roundtrip-efficiency (defaults to 100%)
-            - prefer-charging-sooner (defaults to True)
+            - prefer-charging-sooner (defaults to True, also signals a preference to discharge later)
 
         2) Heat pump sensors are work in progress.
 
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 60de696e9..0e6efe396 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -46,7 +46,6 @@ def initialize_index(
     resolution: timedelta,
     inclusive: str = "left",
 ) -> pd.DatetimeIndex:
-    assert inclusive == "left" or inclusive == "right"
     i = pd.date_range(
         start=start,
         end=end,

From e9ff60bc38742021c1a3bbae315d7ee73dd59d01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Wed, 2 Nov 2022 20:41:18 +0100
Subject: [PATCH 14/44] refactor getting data source for a job to util
 function; use the actual data source ID for this lookup if possible
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/api/v1_3/implementations.py      | 28 ++++------
 flexmeasures/api/v3_0/sensors.py              | 24 +++------
 .../api/v3_0/tests/test_sensor_schedules.py   | 19 +++----
 flexmeasures/data/services/scheduling.py      | 54 ++++++++++++++++---
 4 files changed, 76 insertions(+), 49 deletions(-)

diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index 5db1dda98..19717b9ad 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -39,11 +39,13 @@
     parse_isodate_str,
 )
 from flexmeasures.data import db
-from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.services.resources import has_assets, can_access_asset
-from flexmeasures.data.services.scheduling import create_scheduling_job
+from flexmeasures.data.services.scheduling import (
+    create_scheduling_job,
+    get_data_source_for_job,
+)
 from flexmeasures.utils.time_utils import duration_isoformat
 
 
@@ -145,25 +147,15 @@ def get_device_message_response(generic_asset_name_groups, duration):
                     return unknown_schedule("Scheduling job has an unknown status.")
                 schedule_start = job.kwargs["start"]
 
-            data_source_info = None
-            if job:
-                data_source_info = job.meta.get("data_source_info")
-            if data_source_info is None:
-                data_source_info = dict(
-                    name="Seita"
-                )  # TODO: change to raise later - all scheduling jobs now get full info
-            scheduler_sources = DataSource.query.filter_by(
-                type="scheduling script",
-                **data_source_info,
-            ).all()  # Might be more than one, e.g. per user
-            if len(scheduler_sources) == 0:
-                s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
-                return unknown_schedule(message + f"no data is known from [{s_info}].")
-
+            data_source = get_data_source_for_job(job, sensor=sensor)
+            if data_source is None:
+                return unknown_schedule(
+                    message + f"no data source could be found for job {job}."
+                )
             power_values = sensor.search_beliefs(
                 event_starts_after=schedule_start,
                 event_ends_before=schedule_start + planning_horizon,
-                source=scheduler_sources[-1],
+                source=data_source,
                 most_recent_beliefs_only=True,
                 one_deterministic_belief_per_event=True,
             )
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index a167d7fbb..1e47ac9cc 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -34,7 +34,6 @@
 from flexmeasures.api.common.schemas.users import AccountIdField
 from flexmeasures.api.common.utils.api_utils import save_and_enqueue
 from flexmeasures.auth.decorators import permission_required_for_context
-from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.user import Account
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
@@ -42,7 +41,10 @@
 from flexmeasures.data.schemas.units import QuantityField
 from flexmeasures.data.schemas import AwareDateTimeField
 from flexmeasures.data.services.sensors import get_sensors
-from flexmeasures.data.services.scheduling import create_scheduling_job
+from flexmeasures.data.services.scheduling import (
+    create_scheduling_job,
+    get_data_source_for_job,
+)
 from flexmeasures.utils.time_utils import duration_isoformat
 from flexmeasures.utils.unit_utils import ur
 
@@ -552,25 +554,15 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
             return unknown_schedule("Scheduling job has an unknown status.")
         schedule_start = job.kwargs["start"]
 
-        data_source_info = job.meta.get("data_source_info")
-        if data_source_info is None:
-            data_source_info = dict(
-                name="Seita"
-            )  # TODO: change to raise later - all scheduling jobs now get full info
-        scheduler_sources = DataSource.query.filter_by(
-            type="scheduling script",
-            **data_source_info,
-        ).all()  # there can be more than one, e.g. different users
-        if len(scheduler_sources) == 0:
-            s_info = ",".join([f"{k}={v}" for k, v in data_source_info.items()])
+        data_source = get_data_source_for_job(job, sensor=sensor)
+        if data_source is None:
             return unknown_schedule(
-                error_message + f"no data is known from [{s_info}]."
+                error_message + f"no data source could be found for {data_source}."
             )
-
         power_values = sensor.search_beliefs(
             event_starts_after=schedule_start,
             event_ends_before=schedule_start + planning_horizon,
-            source=scheduler_sources[-1],
+            source=data_source,
             most_recent_beliefs_only=True,
             one_deterministic_belief_per_event=True,
         )
diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
index 5a94668ab..89a5c7e84 100644
--- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
+++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py
@@ -9,10 +9,12 @@
 from flexmeasures.api.tests.utils import get_auth_token
 from flexmeasures.api.v1_3.tests.utils import message_for_get_device_message
 from flexmeasures.api.v3_0.tests.utils import message_for_post_udi_event
-from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.tests.utils import work_on_rq
-from flexmeasures.data.services.scheduling import handle_scheduling_exception
+from flexmeasures.data.services.scheduling import (
+    handle_scheduling_exception,
+    get_data_source_for_job,
+)
 from flexmeasures.utils.calculations import integrate_time_series
 
 
@@ -66,14 +68,13 @@ def test_trigger_and_get_schedule(
     )
 
     # check results are in the database
+
+    # First, make sure the scheduler data source is now there
     job.refresh()  # catch meta info that was added on this very instance
-    data_source_info = job.meta.get("data_source_info")
-    scheduler_source = DataSource.query.filter_by(
-        type="scheduling script", **data_source_info
-    ).one_or_none()
-    assert (
-        scheduler_source is not None
-    )  # Make sure the scheduler data source is now there
+    scheduler_source = get_data_source_for_job(job)
+    assert scheduler_source is not None
+
+    # Then, check if the data was created
     power_values = (
         TimedBelief.query.filter(TimedBelief.sensor_id == sensor.id)
         .filter(TimedBelief.source_id == scheduler_source.id)
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index cd41959f5..7cd54fd22 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -4,11 +4,11 @@
 import sys
 import importlib.util
 from importlib.abc import Loader
+from rq.job import Job
 
 from flask import current_app
 import click
 from rq import get_current_job
-from rq.job import Job
 import timely_beliefs as tb
 
 from flexmeasures.data import db
@@ -16,6 +16,7 @@
 from flexmeasures.data.models.planning.charging_station import schedule_charging_station
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
+from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.utils import get_data_source, save_to_db
 
 """
@@ -145,11 +146,6 @@ def make_schedule(
             % sensor.generic_asset.generic_asset_type
         )
 
-    # saving info on the job, so the API for a job can look the data up
-    if rq_job:
-        rq_job.meta["data_source_info"] = data_source_info
-        rq_job.save_meta()
-
     consumption_schedule = scheduler(
         sensor,
         start,
@@ -171,6 +167,12 @@ def make_schedule(
         data_source_type="scheduling script",
     )
 
+    # saving info on the job, so the API for a job can look the data up
+    data_source_info["id"] = data_source.id
+    if rq_job:
+        rq_job.meta["data_source_info"] = data_source_info
+        rq_job.save_meta()
+
     ts_value_schedule = [
         TimedBelief(
             event_start=dt,
@@ -252,3 +254,43 @@ def handle_scheduling_exception(job, exc_type, exc_value, traceback):
     click.echo("HANDLING RQ WORKER EXCEPTION: %s:%s\n" % (exc_type, exc_value))
     job.meta["exception"] = exc_value
     job.save_meta()
+
+
+def get_data_source_for_job(
+    job: Optional[Job], sensor: Optional[Sensor] = None
+) -> Optional[DataSource]:
+    """
+    Try to find the data source linked by this scheduling job.
+
+    We expect that enough info on the source was placed in the meta dict.
+    For a transition period, we might have to guess a bit.
+    TODO: Afterwards, this can be lighter. We should also expect a job and no sensor is needed,
+          once API v1.3 is deprecated.
+    """
+    data_source_info = None
+    if job:
+        data_source_info = job.meta.get("data_source_info")
+        if data_source_info and "id" in data_source_info:
+            return DataSource.query.get(data_source_info["id"])
+    if data_source_info is None and sensor:
+        data_source_info = dict(
+            name="Seita",
+            model="schedule_battery"
+            if sensor.generic_asset.generic_asset_type.name == "battery"
+            else "schedule_charging_station",
+        )
+        # TODO: change to raise later (v0.13) - all scheduling jobs now get full info
+        current_app.logger.warning(
+            "Looking up scheduling data without knowing full data_source_info (version). This is deprecated soon. Please specify a job id as event or switch to API v3."
+        )
+    scheduler_sources = (
+        DataSource.query.filter_by(
+            type="scheduling script",
+            **data_source_info,
+        )
+        .order_by(DataSource.version.desc())
+        .all()
+    )  # Might still be more than one, e.g. per user
+    if len(scheduler_sources) == 0:
+        return None
+    return scheduler_sources[0]

From 5fe72dcb2f50e8ebb760376ffaf2a32894cdcc33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Wed, 2 Nov 2022 22:14:50 +0100
Subject: [PATCH 15/44] pass sensor to check_storage_specs, as we always have
 it already
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 flexmeasures/api/v1_2/implementations.py               |  2 +-
 flexmeasures/api/v1_3/implementations.py               |  2 +-
 flexmeasures/api/v3_0/sensors.py                       |  2 +-
 flexmeasures/cli/data_add.py                           |  2 +-
 flexmeasures/data/models/planning/tests/test_solver.py | 10 +++++-----
 flexmeasures/data/models/planning/utils.py             |  4 +---
 flexmeasures/data/services/scheduling.py               |  8 ++++----
 flexmeasures/data/tests/test_scheduling_jobs.py        |  4 ++--
 .../data/tests/test_scheduling_jobs_fresh_db.py        |  2 +-
 9 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py
index 647586672..244d54873 100644
--- a/flexmeasures/api/v1_2/implementations.py
+++ b/flexmeasures/api/v1_2/implementations.py
@@ -103,7 +103,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
                 prefer_charging_sooner=False,
             )
             storage_specs = ensure_storage_specs(
-                storage_specs, sensor_id, start, end, resolution
+                storage_specs, sensor, start, end, resolution
             )
             try:
                 schedule = schedule_battery(
diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index 19717b9ad..fed2b0856 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -358,7 +358,7 @@ def post_udi_event_response(unit: str, prior: datetime):
             soc_targets.loc[target_datetime] = target_value
 
     create_scheduling_job(
-        sensor_id,
+        sensor,
         start_of_schedule,
         end_of_schedule,
         resolution=resolution,
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index 1e47ac9cc..40cc2fb22 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -439,7 +439,7 @@ def trigger_schedule(  # noqa: C901
             soc_targets.loc[target_datetime] = target_value
 
         job = create_scheduling_job(
-            sensor.id,
+            sensor,
             start_of_schedule,
             end_of_schedule,
             resolution=resolution,
diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py
index 08a7f345f..458881fe0 100755
--- a/flexmeasures/cli/data_add.py
+++ b/flexmeasures/cli/data_add.py
@@ -953,7 +953,7 @@ def create_schedule(
 
     if as_job:
         job = create_scheduling_job(
-            sensor_id=power_sensor.id,
+            sensor=power_sensor,
             start_of_schedule=start,
             end_of_schedule=end,
             belief_time=server_now(),
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index a53126424..43975c8d8 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -28,7 +28,7 @@ def test_battery_solver_day_1(
     resolution = timedelta(minutes=15)
     soc_at_start = battery.get_attribute("soc_in_mwh")
     storage_specs = ensure_storage_specs(
-        dict(soc_at_start=soc_at_start), battery.id, start, end, resolution
+        dict(soc_at_start=soc_at_start), battery, start, end, resolution
     )
     schedule = schedule_battery(
         battery,
@@ -91,7 +91,7 @@ def test_battery_solver_day_2(add_battery_assets, roundtrip_efficiency: float):
             soc_max=soc_max,
             roundtrip_efficiency=roundtrip_efficiency,
         ),
-        battery.id,
+        battery,
         start,
         end,
         resolution,
@@ -172,7 +172,7 @@ def test_charging_station_solver_day_2(target_soc, charging_station_name):
     soc_targets.loc[target_soc_datetime] = target_soc
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
-        charging_station.id,
+        charging_station,
         start,
         end,
         resolution,
@@ -236,7 +236,7 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
     soc_targets.loc[target_soc_datetime] = target_soc
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
-        charging_station.id,
+        charging_station,
         start,
         end,
         resolution,
@@ -300,7 +300,7 @@ def test_building_solver_day_2(
             soc_min=soc_min,
             soc_max=soc_max,
         ),
-        battery.id,
+        battery,
         start,
         end,
         resolution,
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 0e6efe396..008b70ff9 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -58,7 +58,7 @@ def initialize_index(
 
 def ensure_storage_specs(
     specs: Optional[dict],
-    sensor_id: int,
+    sensor: Sensor,
     start_of_schedule: datetime,
     end_of_schedule: datetime,
     resolution: timedelta,
@@ -77,8 +77,6 @@ def ensure_storage_specs(
     if specs is None:
         specs = {}
 
-    sensor: Optional[Sensor] = Sensor.query.filter_by(id=sensor_id).one_or_none()
-
     # Check state of charge
     # Preferably, a starting soc is given.
     # Otherwise, we try to retrieve the current state of charge from the asset (if that is the valid one at the start).
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 7cd54fd22..ec965980c 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -29,7 +29,7 @@
 
 
 def create_scheduling_job(
-    sensor_id: int,
+    sensor: Sensor,
     start_of_schedule: datetime,
     end_of_schedule: datetime,
     belief_time: datetime,
@@ -44,7 +44,7 @@ def create_scheduling_job(
     """
     Create a new Job, which is queued for later execution.
 
-    Before enqueing, we perform some checks on sensor type and specs, for errors we want to bubble up early.
+    Before enqueuing, we perform some checks on sensor type and specs, for errors we want to bubble up early.
 
     To support quick retrieval of the scheduling job, the job id is the unique entity address of the UDI event.
     That means one event leads to one job (i.e. actions are event driven).
@@ -59,13 +59,13 @@ def create_scheduling_job(
         Freq: 15T, dtype: float64
     """
     storage_specs = ensure_storage_specs(
-        storage_specs, sensor_id, start_of_schedule, end_of_schedule, resolution
+        storage_specs, sensor, start_of_schedule, end_of_schedule, resolution
     )
 
     job = Job.create(
         make_schedule,
         kwargs=dict(
-            sensor_id=sensor_id,
+            sensor_id=sensor.id,
             start=start_of_schedule,
             end=end_of_schedule,
             belief_time=belief_time,
diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py
index da32b69e1..524922dad 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs.py
@@ -35,7 +35,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data):
     )  # Make sure the scheduler data source isn't there
 
     job = create_scheduling_job(
-        battery.id, start, end, belief_time=start, resolution=resolution
+        battery, start, end, belief_time=start, resolution=resolution
     )
 
     print("Job: %s" % job.id)
@@ -103,7 +103,7 @@ def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool):
     resolution = timedelta(minutes=15)
 
     job = create_scheduling_job(
-        battery.id, start, end, belief_time=start, resolution=resolution
+        battery, start, end, belief_time=start, resolution=resolution
     )
     print("Job: %s" % job.id)
 
diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
index 272a79f14..d1cacc420 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
@@ -42,7 +42,7 @@ def test_scheduling_a_charging_station(
     )  # Make sure the scheduler data source isn't there
 
     job = create_scheduling_job(
-        charging_station.id,
+        charging_station,
         start,
         end,
         belief_time=start,

From f80171db94d82a4ac4b442331ecf612716cffc92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= <nicolas@seita.nl>
Date: Thu, 3 Nov 2022 23:32:41 +0100
Subject: [PATCH 16/44] wrap Scheduler in classes, unify data source handling a
 bit more
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Nicolas Höning <nicolas@seita.nl>
---
 documentation/plugin/customisation.rst        |  47 +--
 flexmeasures/api/v1_2/implementations.py      |   4 +-
 flexmeasures/data/models/planning/__init__.py |  11 +
 flexmeasures/data/models/planning/battery.py  | 299 ++++++++---------
 .../data/models/planning/charging_station.py  | 302 +++++++++---------
 .../data/models/planning/tests/test_solver.py |  16 +-
 flexmeasures/data/services/scheduling.py      |  67 ++--
 flexmeasures/data/tests/dummy_scheduler.py    |  43 +--
 .../data/tests/test_scheduling_jobs.py        |  10 +-
 9 files changed, 430 insertions(+), 369 deletions(-)

diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst
index 52fe57397..c3a03ad03 100644
--- a/documentation/plugin/customisation.rst
+++ b/documentation/plugin/customisation.rst
@@ -15,8 +15,8 @@ but in the background your custom scheduling algorithm is being used.
 
 Let's walk through an example!
 
-First, we need to write a function which accepts arguments just like the in-built schedulers (their code is `here <https://github.com/FlexMeasures/flexmeasures/tree/main/flexmeasures/data/models/planning>`_).
-The following minimal example gives you an idea of some meta information you can add for labeling your data, as well as the inputs and outputs of such a scheduling function:
+First, we need to write a a class (inhering from the Base Scheduler) with a `schedule` function which accepts arguments just like the in-built schedulers (their code is `here <https://github.com/FlexMeasures/flexmeasures/tree/main/flexmeasures/data/models/planning>`_).
+The following minimal example gives you an idea of some meta information you can add for labelling your data, as well as the inputs and outputs of such a scheduling function:
 
 .. code-block:: python
 
@@ -24,31 +24,36 @@ The following minimal example gives you an idea of some meta information you can
     import pandas as pd
     from pandas.tseries.frequencies import to_offset
     from flexmeasures.data.models.time_series import Sensor
+    from flexmeasures.data.models.planning import Scheduler
 
 
-    __author__ = "My Company"
-    __version__ = "v2"
+    class DummyScheduler(Scheduler):
 
-    def compute_a_schedule(
-        sensor: Sensor,
-        start: datetime,
-        end: datetime,
-        resolution: timedelta,
-        *args,
-        **kwargs
-    ):
-        """
-        Just a dummy scheduler that always plans to consume at maximum capacity.
-        (Schedulers return positive values for consumption, and negative values for production)
-        """
-        return pd.Series(
-            sensor.get_attribute("capacity_in_mw"),
-            index=pd.date_range(start, end, freq=resolution, closed="left"),
-        )
+        __author__ = "My Company"
+        __version__ = "v2"
+
+        def schedule(
+            self,
+            sensor: Sensor,
+            start: datetime,
+            end: datetime,
+            resolution: timedelta,
+            *args,
+            **kwargs
+        ):
+            """
+            Just a dummy scheduler that always plans to consume at maximum capacity.
+            (Schedulers return positive values for consumption, and negative values for production)
+            """
+            return pd.Series(
+                sensor.get_attribute("capacity_in_mw"),
+                index=pd.date_range(start, end, freq=resolution, closed="left"),
+            )
 
 
 .. note:: It's possible to add arguments that describe the asset flexibility and the EMS context in more detail. For example,
           for storage assets we support various state-of-charge parameters. For now, the existing in-built schedulers are the best documentation.
+          We are working on documenting this better, so the learning curve becomes easier.
 
 
 Finally, make your scheduler be the one that FlexMeasures will use for certain sensors:
@@ -60,7 +65,7 @@ Finally, make your scheduler be the one that FlexMeasures will use for certain s
 
     scheduler_specs = {
         "module": "flexmeasures.data.tests.dummy_scheduler",  # or a file path, see note below
-        "function": "compute_a_schedule",
+        "class": "DummyScheduler",
     }
     
     my_sensor = Sensor.query.filter(Sensor.name == "My power sensor on a flexible asset").one_or_none()
diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py
index 244d54873..5ba96f9e5 100644
--- a/flexmeasures/api/v1_2/implementations.py
+++ b/flexmeasures/api/v1_2/implementations.py
@@ -32,7 +32,7 @@
     parse_isodate_str,
 )
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import schedule_battery
+from flexmeasures.data.models.planning.battery import BatteryScheduler
 from flexmeasures.data.models.planning.exceptions import (
     UnknownMarketException,
     UnknownPricesException,
@@ -106,7 +106,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
                 storage_specs, sensor, start, end, resolution
             )
             try:
-                schedule = schedule_battery(
+                schedule = BatteryScheduler().schedule(
                     sensor, start, end, resolution, storage_specs=storage_specs
                 )
             except UnknownPricesException:
diff --git a/flexmeasures/data/models/planning/__init__.py b/flexmeasures/data/models/planning/__init__.py
index e69de29bb..622b2ccf4 100644
--- a/flexmeasures/data/models/planning/__init__.py
+++ b/flexmeasures/data/models/planning/__init__.py
@@ -0,0 +1,11 @@
+from typing import Optional
+import pandas as pd
+
+
+class Scheduler:
+    """
+    Superclass for all FlexMeasures Schedulers
+    """
+
+    def schedule(*args, **kwargs) -> Optional[pd.Series]:
+        return None
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index ab6d6a676..688d7d10a 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -4,6 +4,7 @@
 import pandas as pd
 
 from flexmeasures.data.models.time_series import Sensor
+from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.models.planning.solver import device_scheduler
 from flexmeasures.data.models.planning.utils import (
     initialize_df,
@@ -15,156 +16,162 @@
 )
 
 
-__version__ = "1"
-__author__ = "Seita"
-
-
-def schedule_battery(
-    sensor: Sensor,
-    start: datetime,
-    end: datetime,
-    resolution: timedelta,
-    storage_specs: dict,
-    consumption_price_sensor: Optional[Sensor] = None,
-    production_price_sensor: Optional[Sensor] = None,
-    inflexible_device_sensors: Optional[List[Sensor]] = None,
-    belief_time: Optional[datetime] = None,
-    round_to_decimals: Optional[int] = 6,
-) -> Union[pd.Series, None]:
-    """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time
-    window.
-    For the resulting consumption schedule, consumption is defined as positive values.
-    """
-
-    soc_at_start = storage_specs.get("soc_at_start")
-    soc_targets = storage_specs.get("soc_targets")
-    soc_min = storage_specs.get("soc_min")
-    soc_max = storage_specs.get("soc_max")
-    roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
-    prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
-
-    # Check for required Sensor attributes
-    sensor.check_required_attributes(
-        [
-            ("capacity_in_mw", (float, int)),
-            ("max_soc_in_mwh", (float, int)),
-            ("min_soc_in_mwh", (float, int)),
-        ],
-    )
-
-    # Check for known prices or price forecasts, trimming planning window accordingly
-    up_deviation_prices, (start, end) = get_prices(
-        (start, end),
-        resolution,
-        beliefs_before=belief_time,
-        price_sensor=consumption_price_sensor,
-        sensor=sensor,
-        allow_trimmed_query_window=True,
-    )
-    down_deviation_prices, (start, end) = get_prices(
-        (start, end),
-        resolution,
-        beliefs_before=belief_time,
-        price_sensor=production_price_sensor,
-        sensor=sensor,
-        allow_trimmed_query_window=True,
-    )
-
-    start = pd.Timestamp(start).tz_convert("UTC")
-    end = pd.Timestamp(end).tz_convert("UTC")
-
-    # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
-    # We penalise the future with at most 1 per thousand times the price spread.
-    if prefer_charging_sooner:
-        up_deviation_prices = add_tiny_price_slope(up_deviation_prices, "event_value")
-        down_deviation_prices = add_tiny_price_slope(
-            down_deviation_prices, "event_value"
+class BatteryScheduler(Scheduler):
+
+    __version__ = "1"
+    __author__ = "Seita"
+
+    def schedule(
+        self,
+        sensor: Sensor,
+        start: datetime,
+        end: datetime,
+        resolution: timedelta,
+        storage_specs: dict,
+        consumption_price_sensor: Optional[Sensor] = None,
+        production_price_sensor: Optional[Sensor] = None,
+        inflexible_device_sensors: Optional[List[Sensor]] = None,
+        belief_time: Optional[datetime] = None,
+        round_to_decimals: Optional[int] = 6,
+    ) -> Union[pd.Series, None]:
+        """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time
+        window.
+        For the resulting consumption schedule, consumption is defined as positive values.
+        """
+
+        soc_at_start = storage_specs.get("soc_at_start")
+        soc_targets = storage_specs.get("soc_targets")
+        soc_min = storage_specs.get("soc_min")
+        soc_max = storage_specs.get("soc_max")
+        roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+        prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
+
+        # Check for required Sensor attributes
+        sensor.check_required_attributes(
+            [
+                ("capacity_in_mw", (float, int)),
+                ("max_soc_in_mwh", (float, int)),
+                ("min_soc_in_mwh", (float, int)),
+            ],
         )
 
-    # Set up commitments to optimise for
-    commitment_quantities = [initialize_series(0, start, end, resolution)]
-
-    # Todo: convert to EUR/(deviation of commitment, which is in MW)
-    commitment_upwards_deviation_price = [
-        up_deviation_prices.loc[start : end - resolution]["event_value"]
-    ]
-    commitment_downwards_deviation_price = [
-        down_deviation_prices.loc[start : end - resolution]["event_value"]
-    ]
-
-    # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
-    columns = [
-        "equals",
-        "max",
-        "min",
-        "derivative equals",
-        "derivative max",
-        "derivative min",
-        "derivative down efficiency",
-        "derivative up efficiency",
-    ]
-    if inflexible_device_sensors is None:
-        inflexible_device_sensors = []
-    device_constraints = [
-        initialize_df(columns, start, end, resolution)
-        for i in range(1 + len(inflexible_device_sensors))
-    ]
-    for i, inflexible_sensor in enumerate(inflexible_device_sensors):
-        device_constraints[i + 1]["derivative equals"] = get_power_values(
-            query_window=(start, end),
-            resolution=resolution,
+        # Check for known prices or price forecasts, trimming planning window accordingly
+        up_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
             beliefs_before=belief_time,
-            sensor=inflexible_sensor,
+            price_sensor=consumption_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
         )
-    if soc_targets is not None:
-        soc_targets = soc_targets.tz_convert("UTC")
-        device_constraints[0]["equals"] = soc_targets.shift(
-            -1, freq=resolution
-        ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
-            timedelta(hours=1) / resolution
-        )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
-        # while the "equals" constraint defines what the total stock should be at the end of a time slot,
-        # where the time slot is indexed by its starting time)
-    device_constraints[0]["min"] = (soc_min - soc_at_start) * (
-        timedelta(hours=1) / resolution
-    )
-    device_constraints[0]["max"] = (soc_max - soc_at_start) * (
-        timedelta(hours=1) / resolution
-    )
-    device_constraints[0]["derivative min"] = (
-        sensor.get_attribute("capacity_in_mw") * -1
-    )
-    device_constraints[0]["derivative max"] = sensor.get_attribute("capacity_in_mw")
-
-    # Apply round-trip efficiency evenly to charging and discharging
-    device_constraints[0]["derivative down efficiency"] = roundtrip_efficiency**0.5
-    device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
-
-    # Set up EMS constraints
-    columns = ["derivative max", "derivative min"]
-    ems_constraints = initialize_df(columns, start, end, resolution)
-    ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
-    if ems_capacity is not None:
-        ems_constraints["derivative min"] = ems_capacity * -1
-        ems_constraints["derivative max"] = ems_capacity
-
-    ems_schedule, expected_costs, scheduler_results = device_scheduler(
-        device_constraints,
-        ems_constraints,
-        commitment_quantities,
-        commitment_downwards_deviation_price,
-        commitment_upwards_deviation_price,
-    )
-    if scheduler_results.solver.termination_condition == "infeasible":
-        # Fallback policy if the problem was unsolvable
-        battery_schedule = fallback_charging_policy(
-            sensor, device_constraints[0], start, end, resolution
+        down_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
+            beliefs_before=belief_time,
+            price_sensor=production_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
         )
-    else:
-        battery_schedule = ems_schedule[0]
 
-    # Round schedule
-    if round_to_decimals:
-        battery_schedule = battery_schedule.round(round_to_decimals)
+        start = pd.Timestamp(start).tz_convert("UTC")
+        end = pd.Timestamp(end).tz_convert("UTC")
+
+        # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
+        # We penalise the future with at most 1 per thousand times the price spread.
+        if prefer_charging_sooner:
+            up_deviation_prices = add_tiny_price_slope(
+                up_deviation_prices, "event_value"
+            )
+            down_deviation_prices = add_tiny_price_slope(
+                down_deviation_prices, "event_value"
+            )
+
+        # Set up commitments to optimise for
+        commitment_quantities = [initialize_series(0, start, end, resolution)]
+
+        # Todo: convert to EUR/(deviation of commitment, which is in MW)
+        commitment_upwards_deviation_price = [
+            up_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+        commitment_downwards_deviation_price = [
+            down_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+
+        # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
+        columns = [
+            "equals",
+            "max",
+            "min",
+            "derivative equals",
+            "derivative max",
+            "derivative min",
+            "derivative down efficiency",
+            "derivative up efficiency",
+        ]
+        if inflexible_device_sensors is None:
+            inflexible_device_sensors = []
+        device_constraints = [
+            initialize_df(columns, start, end, resolution)
+            for i in range(1 + len(inflexible_device_sensors))
+        ]
+        for i, inflexible_sensor in enumerate(inflexible_device_sensors):
+            device_constraints[i + 1]["derivative equals"] = get_power_values(
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                sensor=inflexible_sensor,
+            )
+        if soc_targets is not None:
+            soc_targets = soc_targets.tz_convert("UTC")
+            device_constraints[0]["equals"] = soc_targets.shift(
+                -1, freq=resolution
+            ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
+                timedelta(hours=1) / resolution
+            )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
+            # while the "equals" constraint defines what the total stock should be at the end of a time slot,
+            # where the time slot is indexed by its starting time)
+        device_constraints[0]["min"] = (soc_min - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
+        device_constraints[0]["max"] = (soc_max - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
+        device_constraints[0]["derivative min"] = (
+            sensor.get_attribute("capacity_in_mw") * -1
+        )
+        device_constraints[0]["derivative max"] = sensor.get_attribute("capacity_in_mw")
 
-    return battery_schedule
+        # Apply round-trip efficiency evenly to charging and discharging
+        device_constraints[0]["derivative down efficiency"] = (
+            roundtrip_efficiency**0.5
+        )
+        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
+
+        # Set up EMS constraints
+        columns = ["derivative max", "derivative min"]
+        ems_constraints = initialize_df(columns, start, end, resolution)
+        ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
+        if ems_capacity is not None:
+            ems_constraints["derivative min"] = ems_capacity * -1
+            ems_constraints["derivative max"] = ems_capacity
+
+        ems_schedule, expected_costs, scheduler_results = device_scheduler(
+            device_constraints,
+            ems_constraints,
+            commitment_quantities,
+            commitment_downwards_deviation_price,
+            commitment_upwards_deviation_price,
+        )
+        if scheduler_results.solver.termination_condition == "infeasible":
+            # Fallback policy if the problem was unsolvable
+            battery_schedule = fallback_charging_policy(
+                sensor, device_constraints[0], start, end, resolution
+            )
+        else:
+            battery_schedule = ems_schedule[0]
+
+        # Round schedule
+        if round_to_decimals:
+            battery_schedule = battery_schedule.round(round_to_decimals)
+
+        return battery_schedule
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 8ba57be47..9e88c420b 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -4,6 +4,7 @@
 import pandas as pd
 
 from flexmeasures.data.models.time_series import Sensor
+from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.models.planning.solver import device_scheduler
 from flexmeasures.data.models.planning.utils import (
     initialize_df,
@@ -14,154 +15,165 @@
     fallback_charging_policy,
 )
 
-__version__ = "1"
-__author__ = "Seita"
-
-
-def schedule_charging_station(
-    sensor: Sensor,
-    start: datetime,
-    end: datetime,
-    resolution: timedelta,
-    storage_specs: dict,
-    consumption_price_sensor: Optional[Sensor] = None,
-    production_price_sensor: Optional[Sensor] = None,
-    inflexible_device_sensors: Optional[List[Sensor]] = None,
-    belief_time: Optional[datetime] = None,
-    round_to_decimals: Optional[int] = 6,
-) -> Union[pd.Series, None]:
-    """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time
-    window.
-    For the resulting consumption schedule, consumption is defined as positive values.
-    Todo: handle uni-directional charging by setting the "min" or "derivative min" constraint to 0
-    """
-
-    soc_at_start = storage_specs.get("soc_at_start")
-    soc_targets = storage_specs.get("soc_targets")
-    soc_min = storage_specs.get("soc_min")
-    soc_max = storage_specs.get("soc_max")
-    roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
-    prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
-
-    # Check for required Sensor attributes
-    sensor.check_required_attributes([("capacity_in_mw", (float, int))])
-
-    # Check for known prices or price forecasts, trimming planning window accordingly
-    up_deviation_prices, (start, end) = get_prices(
-        (start, end),
-        resolution,
-        beliefs_before=belief_time,
-        price_sensor=consumption_price_sensor,
-        sensor=sensor,
-        allow_trimmed_query_window=True,
-    )
-    down_deviation_prices, (start, end) = get_prices(
-        (start, end),
-        resolution,
-        beliefs_before=belief_time,
-        price_sensor=production_price_sensor,
-        sensor=sensor,
-        allow_trimmed_query_window=True,
-    )
-
-    soc_targets = soc_targets.tz_convert("UTC")
-    start = pd.Timestamp(start).tz_convert("UTC")
-    end = pd.Timestamp(end).tz_convert("UTC")
-
-    # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
-    # We penalise the future with at most 1 per thousand times the price spread.
-    if prefer_charging_sooner:
-        up_deviation_prices = add_tiny_price_slope(up_deviation_prices, "event_value")
-        down_deviation_prices = add_tiny_price_slope(
-            down_deviation_prices, "event_value"
-        )
 
-    # Set up commitments to optimise for
-    commitment_quantities = [initialize_series(0, start, end, resolution)]
-
-    # Todo: convert to EUR/(deviation of commitment, which is in MW)
-    commitment_upwards_deviation_price = [
-        up_deviation_prices.loc[start : end - resolution]["event_value"]
-    ]
-    commitment_downwards_deviation_price = [
-        down_deviation_prices.loc[start : end - resolution]["event_value"]
-    ]
-
-    # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
-    columns = [
-        "equals",
-        "max",
-        "min",
-        "derivative equals",
-        "derivative max",
-        "derivative min",
-    ]
-    if inflexible_device_sensors is None:
-        inflexible_device_sensors = []
-    device_constraints = [initialize_df(columns, start, end, resolution)] * (
-        1 + len(inflexible_device_sensors)
-    )
-    for i, inflexible_sensor in enumerate(inflexible_device_sensors):
-        device_constraints[i + 1]["derivative equals"] = get_power_values(
-            query_window=(start, end),
-            resolution=resolution,
+class ChargingStationScheduler(Scheduler):
+
+    __version__ = "1"
+    __author__ = "Seita"
+
+    def schedule(
+        self,
+        sensor: Sensor,
+        start: datetime,
+        end: datetime,
+        resolution: timedelta,
+        storage_specs: dict,
+        consumption_price_sensor: Optional[Sensor] = None,
+        production_price_sensor: Optional[Sensor] = None,
+        inflexible_device_sensors: Optional[List[Sensor]] = None,
+        belief_time: Optional[datetime] = None,
+        round_to_decimals: Optional[int] = 6,
+    ) -> Union[pd.Series, None]:
+        """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time
+        window.
+        For the resulting consumption schedule, consumption is defined as positive values.
+        Todo: handle uni-directional charging by setting the "min" or "derivative min" constraint to 0
+        """
+
+        soc_at_start = storage_specs.get("soc_at_start")
+        soc_targets = storage_specs.get("soc_targets")
+        soc_min = storage_specs.get("soc_min")
+        soc_max = storage_specs.get("soc_max")
+        roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+        prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
+
+        # Check for required Sensor attributes
+        sensor.check_required_attributes([("capacity_in_mw", (float, int))])
+
+        # Check for known prices or price forecasts, trimming planning window accordingly
+        up_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
             beliefs_before=belief_time,
-            sensor=inflexible_sensor,
-        )
-    device_constraints[0]["equals"] = soc_targets.shift(-1, freq=resolution).values * (
-        timedelta(hours=1) / resolution
-    ) - soc_at_start * (
-        timedelta(hours=1) / resolution
-    )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
-    # while the "equals" constraint defines what the total stock should be at the end of a time slot,
-    # where the time slot is indexed by its starting time)
-    device_constraints[0]["min"] = (soc_min - soc_at_start) * (
-        timedelta(hours=1) / resolution
-    )
-    device_constraints[0]["max"] = (soc_max - soc_at_start) * (
-        timedelta(hours=1) / resolution
-    )
-
-    if sensor.get_attribute("is_strictly_non_positive"):
-        device_constraints[0]["derivative min"] = 0
-    else:
-        device_constraints[0]["derivative min"] = (
-            sensor.get_attribute("capacity_in_mw") * -1
+            price_sensor=consumption_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
         )
-    if sensor.get_attribute("is_strictly_non_negative"):
-        device_constraints[0]["derivative max"] = 0
-    else:
-        device_constraints[0]["derivative max"] = sensor.get_attribute("capacity_in_mw")
-
-    # Apply round-trip efficiency evenly to charging and discharging
-    device_constraints[0]["derivative down efficiency"] = roundtrip_efficiency**0.5
-    device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
-
-    # Set up EMS constraints
-    columns = ["derivative max", "derivative min"]
-    ems_constraints = initialize_df(columns, start, end, resolution)
-    ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
-    if ems_capacity is not None:
-        ems_constraints["derivative min"] = ems_capacity * -1
-        ems_constraints["derivative max"] = ems_capacity
-
-    ems_schedule, expected_costs, scheduler_results = device_scheduler(
-        device_constraints,
-        ems_constraints,
-        commitment_quantities,
-        commitment_downwards_deviation_price,
-        commitment_upwards_deviation_price,
-    )
-    if scheduler_results.solver.termination_condition == "infeasible":
-        # Fallback policy if the problem was unsolvable
-        charging_station_schedule = fallback_charging_policy(
-            sensor, device_constraints[0], start, end, resolution
+        down_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
+            beliefs_before=belief_time,
+            price_sensor=production_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
         )
-    else:
-        charging_station_schedule = ems_schedule[0]
 
-    # Round schedule
-    if round_to_decimals:
-        charging_station_schedule = charging_station_schedule.round(round_to_decimals)
+        soc_targets = soc_targets.tz_convert("UTC")
+        start = pd.Timestamp(start).tz_convert("UTC")
+        end = pd.Timestamp(end).tz_convert("UTC")
+
+        # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
+        # We penalise the future with at most 1 per thousand times the price spread.
+        if prefer_charging_sooner:
+            up_deviation_prices = add_tiny_price_slope(
+                up_deviation_prices, "event_value"
+            )
+            down_deviation_prices = add_tiny_price_slope(
+                down_deviation_prices, "event_value"
+            )
+
+        # Set up commitments to optimise for
+        commitment_quantities = [initialize_series(0, start, end, resolution)]
+
+        # Todo: convert to EUR/(deviation of commitment, which is in MW)
+        commitment_upwards_deviation_price = [
+            up_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+        commitment_downwards_deviation_price = [
+            down_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+
+        # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
+        columns = [
+            "equals",
+            "max",
+            "min",
+            "derivative equals",
+            "derivative max",
+            "derivative min",
+        ]
+        if inflexible_device_sensors is None:
+            inflexible_device_sensors = []
+        device_constraints = [initialize_df(columns, start, end, resolution)] * (
+            1 + len(inflexible_device_sensors)
+        )
+        for i, inflexible_sensor in enumerate(inflexible_device_sensors):
+            device_constraints[i + 1]["derivative equals"] = get_power_values(
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                sensor=inflexible_sensor,
+            )
+        device_constraints[0]["equals"] = soc_targets.shift(
+            -1, freq=resolution
+        ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
+            timedelta(hours=1) / resolution
+        )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
+        # while the "equals" constraint defines what the total stock should be at the end of a time slot,
+        # where the time slot is indexed by its starting time)
+        device_constraints[0]["min"] = (soc_min - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
+        device_constraints[0]["max"] = (soc_max - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
 
-    return charging_station_schedule
+        if sensor.get_attribute("is_strictly_non_positive"):
+            device_constraints[0]["derivative min"] = 0
+        else:
+            device_constraints[0]["derivative min"] = (
+                sensor.get_attribute("capacity_in_mw") * -1
+            )
+        if sensor.get_attribute("is_strictly_non_negative"):
+            device_constraints[0]["derivative max"] = 0
+        else:
+            device_constraints[0]["derivative max"] = sensor.get_attribute(
+                "capacity_in_mw"
+            )
+
+        # Apply round-trip efficiency evenly to charging and discharging
+        device_constraints[0]["derivative down efficiency"] = (
+            roundtrip_efficiency**0.5
+        )
+        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
+
+        # Set up EMS constraints
+        columns = ["derivative max", "derivative min"]
+        ems_constraints = initialize_df(columns, start, end, resolution)
+        ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
+        if ems_capacity is not None:
+            ems_constraints["derivative min"] = ems_capacity * -1
+            ems_constraints["derivative max"] = ems_capacity
+
+        ems_schedule, expected_costs, scheduler_results = device_scheduler(
+            device_constraints,
+            ems_constraints,
+            commitment_quantities,
+            commitment_downwards_deviation_price,
+            commitment_upwards_deviation_price,
+        )
+        if scheduler_results.solver.termination_condition == "infeasible":
+            # Fallback policy if the problem was unsolvable
+            charging_station_schedule = fallback_charging_policy(
+                sensor, device_constraints[0], start, end, resolution
+            )
+        else:
+            charging_station_schedule = ems_schedule[0]
+
+        # Round schedule
+        if round_to_decimals:
+            charging_station_schedule = charging_station_schedule.round(
+                round_to_decimals
+            )
+
+        return charging_station_schedule
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 43975c8d8..702ae4249 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -6,8 +6,8 @@
 import pandas as pd
 
 from flexmeasures.data.models.time_series import Sensor
-from flexmeasures.data.models.planning.battery import schedule_battery
-from flexmeasures.data.models.planning.charging_station import schedule_charging_station
+from flexmeasures.data.models.planning.battery import BatteryScheduler
+from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.utils.calculations import integrate_time_series
 
@@ -30,7 +30,7 @@ def test_battery_solver_day_1(
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start), battery, start, end, resolution
     )
-    schedule = schedule_battery(
+    schedule = BatteryScheduler().schedule(
         battery,
         start,
         end,
@@ -96,7 +96,7 @@ def test_battery_solver_day_2(add_battery_assets, roundtrip_efficiency: float):
         end,
         resolution,
     )
-    schedule = schedule_battery(
+    schedule = BatteryScheduler().schedule(
         battery,
         start,
         end,
@@ -177,7 +177,7 @@ def test_charging_station_solver_day_2(target_soc, charging_station_name):
         end,
         resolution,
     )
-    consumption_schedule = schedule_charging_station(
+    consumption_schedule = ChargingStationScheduler().schedule(
         charging_station, start, end, resolution, storage_specs=storage_specs
     )
     soc_schedule = integrate_time_series(
@@ -241,7 +241,7 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
         end,
         resolution,
     )
-    consumption_schedule = schedule_charging_station(
+    consumption_schedule = ChargingStationScheduler().schedule(
         charging_station,
         start,
         end,
@@ -305,7 +305,7 @@ def test_building_solver_day_2(
         end,
         resolution,
     )
-    schedule = schedule_battery(
+    schedule = BatteryScheduler().schedule(
         battery,
         start,
         end,
@@ -324,7 +324,7 @@ def test_building_solver_day_2(
         columns=["inflexible"],
     ).tail(
         -4 * 24
-    )  # remove first 96 quarterhours (the schedule is about the 2nd day)
+    )  # remove first 96 quarter-hours (the schedule is about the 2nd day)
     capacity["max"] = building.get_attribute("capacity_in_mw")
     capacity["min"] = -building.get_attribute("capacity_in_mw")
     capacity["production headroom"] = capacity["max"] - capacity["inflexible"]
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index ec965980c..4633fb437 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -1,5 +1,5 @@
 from datetime import datetime, timedelta
-from typing import List, Tuple, Optional, Callable
+from typing import List, Tuple, Optional
 import os
 import sys
 import importlib.util
@@ -12,11 +12,12 @@
 import timely_beliefs as tb
 
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import schedule_battery
-from flexmeasures.data.models.planning.charging_station import schedule_charging_station
+from flexmeasures.data.models.planning.battery import BatteryScheduler
+from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.utils import get_data_source, save_to_db
 
 """
@@ -116,9 +117,6 @@ def make_schedule(
     db.engine.dispose()
 
     sensor = Sensor.query.filter_by(id=sensor_id).one_or_none()
-    data_source_info = dict(
-        name="Seita", model="Unknown", version="1"
-    )  # will be overwritten by scheduler choice below
 
     rq_job = get_current_job()
     if rq_job:
@@ -127,26 +125,31 @@ def make_schedule(
             % (rq_job.id, sensor, start, end)
         )
 
-    # Choose which algorithm to use
+    data_source_info = {}
+    # Choose which algorithm to use  TODO: unify loading this into a func store concept
     if "custom-scheduler" in sensor.attributes:
         scheduler_specs = sensor.attributes.get("custom-scheduler")
         scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
     elif sensor.generic_asset.generic_asset_type.name == "battery":
-        scheduler = schedule_battery
+        scheduler = BatteryScheduler
         data_source_info["model"] = "schedule_battery"
+        data_source_info["name"] = scheduler.__author__
+        data_source_info["version"] = scheduler.__version__
     elif sensor.generic_asset.generic_asset_type.name in (
         "one-way_evse",
         "two-way_evse",
     ):
-        scheduler = schedule_charging_station
+        scheduler = ChargingStationScheduler
         data_source_info["model"] = "schedule_charging_station"
+        data_source_info["name"] = scheduler.__author__
+        data_source_info["version"] = scheduler.__version__
     else:
         raise ValueError(
             "Scheduling is not (yet) supported for asset type %s."
             % sensor.generic_asset.generic_asset_type
         )
 
-    consumption_schedule = scheduler(
+    consumption_schedule = scheduler().schedule(
         sensor,
         start,
         end,
@@ -190,16 +193,19 @@ def make_schedule(
     return True
 
 
-def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, dict]:
+def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Scheduler, dict]:
     """
     Read in custom scheduling spec.
     Attempt to load the Callable, also derive data source info.
 
+    The scheduler class should be derived from flexmeasures.data.models.planning.Scheduler.
+    The Callable is assumed to be named "schedule".
+
     Example specs:
 
     {
         "module": "/path/to/module.py",  # or sthg importable, e.g. "package.module"
-        "function": "name_of_function",
+        "class": "NameOfSchedulerClass",
     }
 
     """
@@ -207,10 +213,12 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, dict]:
         scheduler_specs, dict
     ), f"Scheduler specs is {type(scheduler_specs)}, should be a dict"
     assert "module" in scheduler_specs, "scheduler specs have no 'module'."
-    assert "function" in scheduler_specs, "scheduler specs have no 'function'"
+    assert "class" in scheduler_specs, "scheduler specs have no 'class'"
 
-    scheduler_name = scheduler_specs["function"]
-    source_info = dict(model=scheduler_name, version="1", name="")  # default  # default
+    scheduler_name = scheduler_specs["class"]
+    source_info = dict(
+        model=scheduler_name, version="1", name="Unknown author"
+    )  # default
 
     # import module
     module_descr = scheduler_specs["module"]
@@ -236,15 +244,30 @@ def load_custom_scheduler(scheduler_specs: dict) -> Tuple[Callable, dict]:
 
     # get scheduling function
     assert hasattr(
-        module, scheduler_specs["function"]
-    ), "Module at {module_descr} has no function {scheduler_specs['function']}"
+        module, scheduler_specs["class"]
+    ), "Module at {module_descr} has no class {scheduler_specs['class']}"
 
-    if hasattr(module, "__version__"):
-        source_info["version"] = str(module.__version__)
-    if hasattr(module, "__author__"):
-        source_info["name"] = str(module.__author__)
+    scheduler_class = getattr(module, scheduler_specs["class"])
 
-    return getattr(module, scheduler_specs["function"]), source_info
+    if hasattr(scheduler_class, "__version__"):
+        source_info["version"] = str(scheduler_class.__version__)
+    else:
+        current_app.logger.warning(
+            f"Scheduler {scheduler_class.__name__} loaded, but has no __version__ attribute."
+        )
+    if hasattr(scheduler_class, "__author__"):
+        source_info["name"] = str(scheduler_class.__author__)
+    else:
+        current_app.logger.warning(
+            f"Scheduler {scheduler_class.__name__} loaded, but has no __author__ attribute."
+        )
+
+    schedule_function_name = "schedule"
+    if not hasattr(scheduler_class, schedule_function_name):
+        raise NotImplementedError(
+            f"No function {schedule_function_name} in {scheduler_class}. Cannot load custom scheduler."
+        )
+    return scheduler_class, source_info
 
 
 def handle_scheduling_exception(job, exc_type, exc_value, traceback):
diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py
index b9193b440..19908dee3 100644
--- a/flexmeasures/data/tests/dummy_scheduler.py
+++ b/flexmeasures/data/tests/dummy_scheduler.py
@@ -1,28 +1,31 @@
 from datetime import datetime, timedelta
 
 from flexmeasures.data.models.time_series import Sensor
+from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.models.planning.utils import initialize_series
 
 
-__author__ = "Test Organization"
-__version__ = "v3"
+class DummyScheduler(Scheduler):
 
+    __author__ = "Test Organization"
+    __version__ = "v3"
 
-def compute_a_schedule(
-    sensor: Sensor,
-    start: datetime,
-    end: datetime,
-    resolution: timedelta,
-    *args,
-    **kwargs
-):
-    """
-    Just a dummy scheduler that always plans to consume at maximum capacity.
-    (Schedulers return positive values for consumption, and negative values for production)
-    """
-    return initialize_series(  # simply creates a Pandas Series repeating one value
-        data=sensor.get_attribute("capacity_in_mw"),
-        start=start,
-        end=end,
-        resolution=resolution,
-    )
+    def schedule(
+        self,
+        sensor: Sensor,
+        start: datetime,
+        end: datetime,
+        resolution: timedelta,
+        *args,
+        **kwargs
+    ):
+        """
+        Just a dummy scheduler that always plans to consume at maximum capacity.
+        (Schedulers return positive values for consumption, and negative values for production)
+        """
+        return initialize_series(  # simply creates a Pandas Series repeating one value
+            data=sensor.get_attribute("capacity_in_mw"),
+            start=start,
+            end=end,
+            resolution=resolution,
+        )
diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py
index 524922dad..676eee92d 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs.py
@@ -60,7 +60,7 @@ def test_scheduling_a_battery(db, app, add_battery_assets, setup_test_data):
 
 scheduler_specs = {
     "module": None,  # use make_module_descr, see below
-    "function": "compute_a_schedule",
+    "class": "DummyScheduler",
 }
 
 
@@ -81,9 +81,9 @@ def test_loading_custom_scheduler(is_path: bool):
     custom_scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
     assert data_source_info["name"] == "Test Organization"
     assert data_source_info["version"] == "v3"
-    assert data_source_info["model"] == "compute_a_schedule"
-    assert custom_scheduler.__name__ == "compute_a_schedule"
-    assert "Just a dummy scheduler" in custom_scheduler.__doc__
+    assert data_source_info["model"] == "DummyScheduler"
+    assert custom_scheduler.__name__ == "DummyScheduler"
+    assert "Just a dummy scheduler" in custom_scheduler.schedule.__doc__
 
 
 @pytest.mark.parametrize("is_path", [False, True])
@@ -112,7 +112,7 @@ def test_assigning_custom_scheduler(db, app, add_battery_assets, is_path: bool):
     # make sure we saved the data source for later lookup
     redis_connection = app.queues["scheduling"].connection
     finished_job = Job.fetch(job.id, connection=redis_connection)
-    assert finished_job.meta["data_source_info"]["model"] == scheduler_specs["function"]
+    assert finished_job.meta["data_source_info"]["model"] == scheduler_specs["class"]
 
     scheduler_source = DataSource.query.filter_by(
         type="scheduling script",

From 22cb85219d11e093e1bf4d091af504d1aa12681e Mon Sep 17 00:00:00 2001
From: Felix Claessen <30658763+Flix6x@users.noreply.github.com>
Date: Thu, 10 Nov 2022 12:44:43 +0100
Subject: [PATCH 17/44] Support pandas 1.4 (#525)

Add a pandas version check in initialize_index.


* Use initialize_series util function

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Update initialize_index for pandas>=1.4

Signed-off-by: F.N. Claessen <felix@seita.nl>

* flake8

Signed-off-by: F.N. Claessen <felix@seita.nl>

* Use initialize_index or initialize_series in all places where the closed keyword argument was used

Signed-off-by: F.N. Claessen <felix@seita.nl>

* flake8

Signed-off-by: F.N. Claessen <felix@seita.nl>

* mypy: PEP 484 prohibits implicit Optional

Signed-off-by: F.N. Claessen <felix@seita.nl>

* black after mypy

Signed-off-by: F.N. Claessen <felix@seita.nl>

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../api/common/schemas/sensor_data.py         |  5 +--
 flexmeasures/api/common/utils/api_utils.py    | 12 +++---
 flexmeasures/api/common/utils/validators.py   |  6 ++-
 flexmeasures/api/v1/implementations.py        |  8 +++-
 flexmeasures/api/v1_3/implementations.py      | 10 +++--
 flexmeasures/api/v3_0/sensors.py              | 11 +++---
 flexmeasures/cli/data_add.py                  | 16 ++++----
 flexmeasures/conftest.py                      | 21 +++++-----
 flexmeasures/data/models/planning/solver.py   |  8 ++--
 .../data/models/planning/tests/conftest.py    | 13 +++----
 .../data/models/planning/tests/test_solver.py | 13 +++----
 flexmeasures/data/models/planning/utils.py    | 38 +++++++++++--------
 flexmeasures/data/queries/annotations.py      |  4 +-
 flexmeasures/data/services/scheduling.py      |  2 +-
 flexmeasures/data/tests/test_queries.py       | 29 ++++++--------
 .../tests/test_scheduling_jobs_fresh_db.py    |  5 +--
 flexmeasures/ui/crud/users.py                 |  4 +-
 flexmeasures/utils/calculations.py            |  8 ++--
 18 files changed, 110 insertions(+), 103 deletions(-)

diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py
index de0fc0ae4..b44fc6738 100644
--- a/flexmeasures/api/common/schemas/sensor_data.py
+++ b/flexmeasures/api/common/schemas/sensor_data.py
@@ -14,6 +14,7 @@
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.api.common.schemas.sensors import SensorField
 from flexmeasures.api.common.utils.api_utils import upsample_values
+from flexmeasures.data.models.planning.utils import initialize_index
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 from flexmeasures.data.services.time_series import simplify_index
 from flexmeasures.utils.time_utils import duration_isoformat, server_now
@@ -179,9 +180,7 @@ def dump_bdf(self, sensor_data_description: dict, **kwargs) -> dict:
         )
 
         # Convert to desired time range
-        index = pd.date_range(
-            start=start, end=end, freq=df.event_resolution, closed="left"
-        )
+        index = initialize_index(start=start, end=end, resolution=df.event_resolution)
         df = df.reindex(index)
 
         # Convert to desired unit
diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py
index d92c6e017..7d36299f0 100644
--- a/flexmeasures/api/common/utils/api_utils.py
+++ b/flexmeasures/api/common/utils/api_utils.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from timely_beliefs.beliefs.classes import BeliefsDataFrame
 from typing import List, Sequence, Tuple, Union
 import copy
@@ -54,8 +56,8 @@ def contains_empty_items(groups: List[List[str]]):
 
 
 def parse_as_list(
-    connection: Union[Sequence[Union[str, float]], str, float], of_type: type = None
-) -> Sequence[Union[str, float, None]]:
+    connection: str | float | Sequence[str | float], of_type: type | None = None
+) -> Sequence[str | float | None]:
     """
     Return a list of connections (or values), even if it's just one connection (or value)
     """
@@ -141,7 +143,7 @@ def groups_to_dict(
     connection_groups: List[str],
     value_groups: List[List[str]],
     generic_asset_type_name: str,
-    plural_name: str = None,
+    plural_name: str | None = None,
     groups_name="groups",
 ) -> dict:
     """Put the connections and values in a dictionary and simplify if groups have identical values and/or if there is
@@ -343,7 +345,7 @@ def get_sensor_by_generic_asset_type_and_location(
 
 
 def enqueue_forecasting_jobs(
-    forecasting_jobs: List[Job] = None,
+    forecasting_jobs: list[Job] | None = None,
 ):
     """Enqueue forecasting jobs.
 
@@ -355,7 +357,7 @@ def enqueue_forecasting_jobs(
 
 def save_and_enqueue(
     data: Union[BeliefsDataFrame, List[BeliefsDataFrame]],
-    forecasting_jobs: List[Job] = None,
+    forecasting_jobs: list[Job] | None = None,
     save_changed_beliefs_only: bool = True,
 ) -> ResponseTuple:
 
diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py
index 7df1ce975..50ce9f22c 100644
--- a/flexmeasures/api/common/utils/validators.py
+++ b/flexmeasures/api/common/utils/validators.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime, timedelta
 from functools import wraps
 from typing import List, Tuple, Union, Optional
@@ -228,7 +230,7 @@ def decorated_service(*args, **kwargs):
 
 
 def optional_user_sources_accepted(
-    default_source: Union[int, str, List[Union[int, str]]] = None
+    default_source: int | str | list[int | str] | None = None,
 ):
     """Decorator which specifies that a GET or POST request accepts an optional source or list of data sources.
     It parses relevant form data and sets the "user_source_ids" keyword parameter.
@@ -539,7 +541,7 @@ def wrapper(*args, **kwargs):
 
 
 def assets_required(
-    generic_asset_type_name: str, plural_name: str = None, groups_name="groups"
+    generic_asset_type_name: str, plural_name: str | None = None, groups_name="groups"
 ):
     """Decorator which specifies that a GET or POST request must specify one or more assets.
     It parses relevant form data and sets the "generic_asset_name_groups" keyword param.
diff --git a/flexmeasures/api/v1/implementations.py b/flexmeasures/api/v1/implementations.py
index 864e51f54..669ca94fb 100644
--- a/flexmeasures/api/v1/implementations.py
+++ b/flexmeasures/api/v1/implementations.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import isodate
 from typing import Dict, List, Optional, Tuple, Union
 from datetime import datetime as datetime_type, timedelta
@@ -154,8 +156,10 @@ def collect_connection_and_value_groups(
     start: datetime_type,
     duration: timedelta,
     connection_groups: List[List[str]],
-    user_source_ids: Union[int, List[int]] = None,  # None is interpreted as all sources
-    source_types: List[str] = None,
+    user_source_ids: int
+    | list[int]
+    | None = None,  # None is interpreted as all sources
+    source_types: list[str] | None = None,
 ) -> Tuple[dict, int]:
     """
     Code for GETting power values from the API.
diff --git a/flexmeasures/api/v1_3/implementations.py b/flexmeasures/api/v1_3/implementations.py
index fed2b0856..88996e16c 100644
--- a/flexmeasures/api/v1_3/implementations.py
+++ b/flexmeasures/api/v1_3/implementations.py
@@ -39,6 +39,7 @@
     parse_isodate_str,
 )
 from flexmeasures.data import db
+from flexmeasures.data.models.planning.utils import initialize_series
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.services.resources import has_assets, can_access_asset
@@ -300,11 +301,12 @@ def post_udi_event_response(unit: str, prior: datetime):
     start_of_schedule = datetime
     end_of_schedule = datetime + current_app.config.get("FLEXMEASURES_PLANNING_HORIZON")
     resolution = sensor.event_resolution
-    soc_targets = pd.Series(
+    soc_targets = initialize_series(
         np.nan,
-        index=pd.date_range(
-            start_of_schedule, end_of_schedule, freq=resolution, closed="right"
-        ),  # note that target values are indexed by their due date (i.e. closed="right")
+        start=start_of_schedule,
+        end=end_of_schedule,
+        resolution=resolution,
+        inclusive="right",  # note that target values are indexed by their due date (i.e. inclusive="right")
     )
 
     if event_type == "soc-with-targets":
diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py
index 40cc2fb22..901d63bb2 100644
--- a/flexmeasures/api/v3_0/sensors.py
+++ b/flexmeasures/api/v3_0/sensors.py
@@ -10,7 +10,6 @@
 from marshmallow import validate, fields, Schema
 from marshmallow.validate import OneOf
 import numpy as np
-import pandas as pd
 from rq.job import Job, NoSuchJobError
 from timely_beliefs import BeliefsDataFrame
 from webargs.flaskparser import use_args, use_kwargs
@@ -34,6 +33,7 @@
 from flexmeasures.api.common.schemas.users import AccountIdField
 from flexmeasures.api.common.utils.api_utils import save_and_enqueue
 from flexmeasures.auth.decorators import permission_required_for_context
+from flexmeasures.data.models.planning.utils import initialize_series
 from flexmeasures.data.models.user import Account
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
@@ -388,11 +388,12 @@ def trigger_schedule(  # noqa: C901
             "FLEXMEASURES_PLANNING_HORIZON"
         )
         resolution = sensor.event_resolution
-        soc_targets = pd.Series(
+        soc_targets = initialize_series(
             np.nan,
-            index=pd.date_range(
-                start_of_schedule, end_of_schedule, freq=resolution, closed="right"
-            ),  # note that target values are indexed by their due date (i.e. closed="right")
+            start=start_of_schedule,
+            end=end_of_schedule,
+            resolution=resolution,
+            inclusive="right",  # note that target values are indexed by their due date (i.e. inclusive="right")
         )
         # todo: move this deserialization of targets into newly-created ScheduleTargetSchema
         for target in kwargs.get("targets", []):
diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py
index 03d29a47d..889b7929a 100755
--- a/flexmeasures/cli/data_add.py
+++ b/flexmeasures/cli/data_add.py
@@ -1,4 +1,5 @@
 """CLI Tasks for populating the database - most useful in development"""
+from __future__ import annotations
 
 from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple
@@ -27,6 +28,7 @@
 from flexmeasures.data.services.forecasting import create_forecasting_jobs
 from flexmeasures.data.services.scheduling import make_schedule, create_scheduling_job
 from flexmeasures.data.services.users import create_user
+from flexmeasures.data.models.planning.utils import initialize_series
 from flexmeasures.data.models.user import Account, AccountRole, RolesAccounts
 from flexmeasures.data.models.time_series import (
     Sensor,
@@ -403,7 +405,7 @@ def add_beliefs(
     resample: bool = True,
     allow_overwrite: bool = False,
     skiprows: int = 1,
-    na_values: List[str] = None,
+    na_values: list[str] | None = None,
     nrows: Optional[int] = None,
     datecol: int = 0,
     valuecol: int = 1,
@@ -923,14 +925,12 @@ def create_schedule(
         except MissingAttributeException:
             click.echo(f"{power_sensor} has no {attribute} attribute.")
             raise click.Abort()
-    soc_targets = pd.Series(
+    soc_targets = initialize_series(
         np.nan,
-        index=pd.date_range(
-            pd.Timestamp(start).tz_convert(power_sensor.timezone),
-            pd.Timestamp(end).tz_convert(power_sensor.timezone),
-            freq=power_sensor.event_resolution,
-            closed="right",
-        ),  # note that target values are indexed by their due date (i.e. closed="right")
+        start=pd.Timestamp(start).tz_convert(power_sensor.timezone),
+        end=pd.Timestamp(end).tz_convert(power_sensor.timezone),
+        resolution=power_sensor.event_resolution,
+        inclusive="right",  # note that target values are indexed by their due date (i.e. inclusive="right")
     )
 
     # Convert round-trip efficiency to dimensionless
diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py
index b8455cb98..c6a613f9c 100644
--- a/flexmeasures/conftest.py
+++ b/flexmeasures/conftest.py
@@ -26,6 +26,7 @@
 from flexmeasures.data.models.assets import AssetType, Asset
 from flexmeasures.data.models.generic_assets import GenericAssetType, GenericAsset
 from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.models.planning.utils import initialize_index
 from flexmeasures.data.models.markets import Market, MarketType
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.user import User, Account, AccountRole
@@ -494,12 +495,10 @@ def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets, setup_sources
     """Add two days of market prices for the EPEX day-ahead market."""
 
     # one day of test data (one complete sine curve)
-    time_slots = pd.date_range(
-        datetime(2015, 1, 1),
-        datetime(2015, 1, 2),
-        freq="1H",
-        closed="left",
-        tz="Europe/Amsterdam",
+    time_slots = initialize_index(
+        start=pd.Timestamp("2015-01-01").tz_localize("Europe/Amsterdam"),
+        end=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"),
+        resolution="1H",
     )
     values = [
         random() * (1 + np.sin(x * 2 * np.pi / 24)) for x in range(len(time_slots))
@@ -517,12 +516,10 @@ def add_market_prices(db: SQLAlchemy, setup_assets, setup_markets, setup_sources
     db.session.add_all(day1_beliefs)
 
     # another day of test data (8 expensive hours, 8 cheap hours, and again 8 expensive hours)
-    time_slots = pd.date_range(
-        datetime(2015, 1, 2),
-        datetime(2015, 1, 3),
-        freq="1H",
-        closed="left",
-        tz="Europe/Amsterdam",
+    time_slots = initialize_index(
+        start=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"),
+        end=pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam"),
+        resolution="1H",
     )
     values = [100] * 8 + [90] * 8 + [100] * 8
     day2_beliefs = [
diff --git a/flexmeasures/data/models/planning/solver.py b/flexmeasures/data/models/planning/solver.py
index 42c8acf29..32d8b7456 100644
--- a/flexmeasures/data/models/planning/solver.py
+++ b/flexmeasures/data/models/planning/solver.py
@@ -323,11 +323,11 @@ def cost_function(m):
     for d in model.d:
         planned_device_power = [model.ems_power[d, j].value for j in model.j]
         planned_power_per_device.append(
-            pd.Series(
-                index=pd.date_range(
-                    start=start, end=end, freq=to_offset(resolution), closed="left"
-                ),
+            initialize_series(
                 data=planned_device_power,
+                start=start,
+                end=end,
+                resolution=to_offset(resolution),
             )
         )
 
diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py
index 89d39ccaf..e59db120f 100644
--- a/flexmeasures/data/models/planning/tests/conftest.py
+++ b/flexmeasures/data/models/planning/tests/conftest.py
@@ -1,11 +1,12 @@
 from __future__ import annotations
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 import pytest
 
 import pandas as pd
 
 from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
+from flexmeasures.data.models.planning.utils import initialize_index
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 
 
@@ -98,12 +99,10 @@ def add_inflexible_device_forecasts(
     Set up inflexible devices and forecasts.
     """
     # 2 days of test data
-    time_slots = pd.date_range(
-        datetime(2015, 1, 1),
-        datetime(2015, 1, 3),
-        freq="15T",
-        closed="left",
-        tz="Europe/Amsterdam",
+    time_slots = initialize_index(
+        start=pd.Timestamp("2015-01-01").tz_localize("Europe/Amsterdam"),
+        end=pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam"),
+        resolution="15T",
     )
 
     # PV (8 hours at zero capacity, 8 hours at 90% capacity, and again 8 hours at zero capacity)
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 0611caa0b..62eb3842e 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -8,7 +8,10 @@
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.models.planning.battery import BatteryScheduler
 from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
-from flexmeasures.data.models.planning.utils import ensure_storage_specs
+from flexmeasures.data.models.planning.utils import (
+    ensure_storage_specs,
+    initialize_series,
+)
 from flexmeasures.utils.calculations import integrate_time_series
 
 
@@ -172,9 +175,7 @@ def test_charging_station_solver_day_2(target_soc, charging_station_name):
     end = tz.localize(datetime(2015, 1, 3))
     resolution = timedelta(minutes=15)
     target_soc_datetime = start + duration_until_target
-    soc_targets = pd.Series(
-        np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
-    )
+    soc_targets = initialize_series(np.nan, start, end, resolution, inclusive="right")
     soc_targets.loc[target_soc_datetime] = target_soc
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
@@ -236,9 +237,7 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
     end = tz.localize(datetime(2015, 1, 3))
     resolution = timedelta(minutes=15)
     target_soc_datetime = start + duration_until_target
-    soc_targets = pd.Series(
-        np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
-    )
+    soc_targets = initialize_series(np.nan, start, end, resolution, inclusive="right")
     soc_targets.loc[target_soc_datetime] = target_soc
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start, soc_targets=soc_targets),
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 008b70ff9..6b7f56c96 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -1,3 +1,4 @@
+from packaging import version
 from typing import List, Optional, Tuple, Union
 from datetime import date, datetime, timedelta
 
@@ -41,19 +42,27 @@ def initialize_series(
 
 
 def initialize_index(
-    start: Union[date, datetime],
-    end: Union[date, datetime],
-    resolution: timedelta,
+    start: Union[date, datetime, str],
+    end: Union[date, datetime, str],
+    resolution: Union[timedelta, str],
     inclusive: str = "left",
 ) -> pd.DatetimeIndex:
-    i = pd.date_range(
-        start=start,
-        end=end,
-        freq=to_offset(resolution),
-        closed=inclusive,
-        name="datetime",
-    )
-    return i
+    if version.parse(pd.__version__) >= version.parse("1.4.0"):
+        return pd.date_range(
+            start=start,
+            end=end,
+            freq=to_offset(resolution),
+            inclusive=inclusive,
+            name="datetime",
+        )
+    else:
+        return pd.date_range(
+            start=start,
+            end=end,
+            freq=to_offset(resolution),
+            closed=inclusive,
+            name="datetime",
+        )
 
 
 def ensure_storage_specs(
@@ -93,11 +102,8 @@ def ensure_storage_specs(
 
     # init default targets
     if "soc_targets" not in specs or specs["soc_targets"] is None:
-        specs["soc_targets"] = pd.Series(
-            np.nan,
-            index=pd.date_range(
-                start_of_schedule, end_of_schedule, freq=resolution, closed="right"
-            ),
+        specs["soc_targets"] = initialize_series(
+            np.nan, start_of_schedule, end_of_schedule, resolution, inclusive="right"
         )
     # soc targets are at the end of each time slot, while prices are indexed by the start of each time slot
     specs["soc_targets"] = specs["soc_targets"][
diff --git a/flexmeasures/data/queries/annotations.py b/flexmeasures/data/queries/annotations.py
index a9b344a8b..dff7ef85f 100644
--- a/flexmeasures/data/queries/annotations.py
+++ b/flexmeasures/data/queries/annotations.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 from typing import List, Optional
 
@@ -15,7 +17,7 @@ def query_asset_annotations(
     annotations_after: Optional[datetime] = None,
     annotations_before: Optional[datetime] = None,
     sources: Optional[List[DataSource]] = None,
-    annotation_type: str = None,
+    annotation_type: str | None = None,
 ) -> Query:
     """Match annotations assigned to the given asset."""
     query = Annotation.query.join(GenericAssetAnnotationRelationship).filter(
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 4633fb437..384839057 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -51,7 +51,7 @@ def create_scheduling_job(
     That means one event leads to one job (i.e. actions are event driven).
 
     Target SOC values should be indexed by their due date. For example, for quarter-hourly targets between 5 and 6 AM:
-    >>> df = pd.Series(data=[1, 2, 2.5, 3], index=pd.date_range(datetime(2010,1,1,5), datetime(2010,1,1,6), freq=timedelta(minutes=15), closed="right"))
+    >>> df = pd.Series(data=[1, 2, 2.5, 3], index=pd.date_range(datetime(2010,1,1,5), datetime(2010,1,1,6), freq=timedelta(minutes=15), inclusive="right"))
     >>> print(df)
         2010-01-01 05:15:00    1.0
         2010-01-01 05:30:00    2.0
diff --git a/flexmeasures/data/tests/test_queries.py b/flexmeasures/data/tests/test_queries.py
index 85068475a..109cc0219 100644
--- a/flexmeasures/data/tests/test_queries.py
+++ b/flexmeasures/data/tests/test_queries.py
@@ -7,6 +7,7 @@
 import timely_beliefs as tb
 
 from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.models.planning.utils import initialize_index
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.queries.utils import (
     multiply_dataframe_with_deterministic_beliefs,
@@ -108,16 +109,12 @@ def test_collect_power_resampled(
 def test_multiplication():
     df1 = pd.DataFrame(
         [[30.0, timedelta(hours=3)]],
-        index=pd.date_range(
-            "2000-01-01 10:00", "2000-01-01 15:00", freq="1h", closed="left"
-        ),
+        index=initialize_index("2000-01-01 10:00", "2000-01-01 15:00", resolution="1h"),
         columns=["event_value", "belief_horizon"],
     )
     df2 = pd.DataFrame(
         [[10.0, timedelta(hours=1)]],
-        index=pd.date_range(
-            "2000-01-01 13:00", "2000-01-01 18:00", freq="1h", closed="left"
-        ),
+        index=initialize_index("2000-01-01 13:00", "2000-01-01 18:00", resolution="1h"),
         columns=["event_value", "belief_horizon"],
     )
     df = multiply_dataframe_with_deterministic_beliefs(df1, df2)
@@ -125,22 +122,22 @@ def test_multiplication():
         [
             pd.DataFrame(
                 [[np.nan, timedelta(hours=3)]],
-                index=pd.date_range(
-                    "2000-01-01 10:00", "2000-01-01 13:00", freq="1h", closed="left"
+                index=initialize_index(
+                    "2000-01-01 10:00", "2000-01-01 13:00", resolution="1h"
                 ),
                 columns=["event_value", "belief_horizon"],
             ),
             pd.DataFrame(
                 [[300.0, timedelta(hours=1)]],
-                index=pd.date_range(
-                    "2000-01-01 13:00", "2000-01-01 15:00", freq="1h", closed="left"
+                index=initialize_index(
+                    "2000-01-01 13:00", "2000-01-01 15:00", resolution="1h"
                 ),
                 columns=["event_value", "belief_horizon"],
             ),
             pd.DataFrame(
                 [[np.nan, timedelta(hours=1)]],
-                index=pd.date_range(
-                    "2000-01-01 15:00", "2000-01-01 18:00", freq="1h", closed="left"
+                index=initialize_index(
+                    "2000-01-01 15:00", "2000-01-01 18:00", resolution="1h"
                 ),
                 columns=["event_value", "belief_horizon"],
             ),
@@ -161,17 +158,13 @@ def test_multiplication_with_one_empty_dataframe():
 
     df2 = pd.DataFrame(
         [[10.0, timedelta(hours=1)]],
-        index=pd.date_range(
-            "2000-01-01 13:00", "2000-01-01 18:00", freq="1h", closed="left"
-        ),
+        index=initialize_index("2000-01-01 13:00", "2000-01-01 18:00", resolution="1h"),
         columns=["event_value", "belief_horizon"],
     )
 
     df_compare = pd.DataFrame(
         [[np.nan, timedelta(hours=1)]],
-        index=pd.date_range(
-            "2000-01-01 13:00", "2000-01-01 18:00", freq="1h", closed="left"
-        ),
+        index=initialize_index("2000-01-01 13:00", "2000-01-01 18:00", resolution="1h"),
         columns=["event_value", "belief_horizon"],
     )
     # set correct types
diff --git a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
index d1cacc420..534435773 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs_fresh_db.py
@@ -5,6 +5,7 @@
 import pandas as pd
 
 from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.data.models.planning.utils import initialize_series
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.services.scheduling import create_scheduling_job
 from flexmeasures.data.tests.utils import work_on_rq, exception_reporter
@@ -31,9 +32,7 @@ def test_scheduling_a_charging_station(
     end = tz.localize(datetime(2015, 1, 3))
     resolution = timedelta(minutes=15)
     target_soc_datetime = start + duration_until_target
-    soc_targets = pd.Series(
-        np.nan, index=pd.date_range(start, end, freq=resolution, closed="right")
-    )
+    soc_targets = initialize_series(np.nan, start, end, resolution, inclusive="right")
     soc_targets.loc[target_soc_datetime] = target_soc
 
     assert (
diff --git a/flexmeasures/ui/crud/users.py b/flexmeasures/ui/crud/users.py
index 8a39ec97c..b09adb934 100644
--- a/flexmeasures/ui/crud/users.py
+++ b/flexmeasures/ui/crud/users.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Optional, Union
 from datetime import datetime
 
@@ -36,7 +38,7 @@ class UserForm(FlaskForm):
     active = BooleanField("Activation Status", validators=[DataRequired()])
 
 
-def render_user(user: Optional[User], asset_count: int = 0, msg: str = None):
+def render_user(user: User | None, asset_count: int = 0, msg: str | None = None):
     user_form = UserForm()
     user_form.process(obj=user)
     return render_flexmeasures_template(
diff --git a/flexmeasures/utils/calculations.py b/flexmeasures/utils/calculations.py
index 84cb8dd79..f85813fa7 100644
--- a/flexmeasures/utils/calculations.py
+++ b/flexmeasures/utils/calculations.py
@@ -44,13 +44,13 @@ def integrate_time_series(
     down_efficiency: float | pd.Series = 1,
     decimal_precision: int | None = None,
 ) -> pd.Series:
-    """Integrate time series of length n and closed="left" (representing a flow)
-    to a time series of length n+1 and closed="both" (representing a stock),
+    """Integrate time series of length n and inclusive="left" (representing a flow)
+    to a time series of length n+1 and inclusive="both" (representing a stock),
     given an initial stock (i.e. the constant of integration).
     The unit of time is hours: i.e. the stock unit is flow unit times hours (e.g. a flow in kW becomes a stock in kWh).
     Optionally, set a decimal precision to round off the results (useful for tests failing over machine precision).
 
-    >>> s = pd.Series([1, 2, 3, 4], index=pd.date_range(datetime(2001, 1, 1, 5), datetime(2001, 1, 1, 6), freq=timedelta(minutes=15), closed="left"))
+    >>> s = pd.Series([1, 2, 3, 4], index=pd.date_range(datetime(2001, 1, 1, 5), datetime(2001, 1, 1, 6), freq=timedelta(minutes=15), inclusive="left"))
     >>> integrate_time_series(s, 10)
         2001-01-01 05:00:00    10.00
         2001-01-01 05:15:00    10.25
@@ -59,7 +59,7 @@ def integrate_time_series(
         2001-01-01 06:00:00    12.50
         Freq: D, dtype: float64
 
-    >>> s = pd.Series([1, 2, 3, 4], index=pd.date_range(datetime(2001, 1, 1, 5), datetime(2001, 1, 1, 7), freq=timedelta(minutes=30), closed="left"))
+    >>> s = pd.Series([1, 2, 3, 4], index=pd.date_range(datetime(2001, 1, 1, 5), datetime(2001, 1, 1, 7), freq=timedelta(minutes=30), inclusive="left"))
     >>> integrate_time_series(s, 10)
         2001-01-01 05:00:00    10.0
         2001-01-01 05:30:00    10.5

From dd47dabd1d1482873359546f0dc156f0c522ac68 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:24:00 +0100
Subject: [PATCH 18/44] Stop requiring min/max SoC attributes, which have
 defaults: - Default min = 0 - Default max = the highest target value, or
 np.nan if there are no targets, which subsequently maps to infinity in our
 solver

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/battery.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 688d7d10a..aef80cdc2 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -50,8 +50,6 @@ def schedule(
         sensor.check_required_attributes(
             [
                 ("capacity_in_mw", (float, int)),
-                ("max_soc_in_mwh", (float, int)),
-                ("min_soc_in_mwh", (float, int)),
             ],
         )
 

From add377f6c459f34530a67ce304c94bcc90d9f5b3 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:26:34 +0100
Subject: [PATCH 19/44] Set up device constraint columns for efficiencies in
 Charge Point scheduler, too

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/charging_station.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 727bed7ff..0128876a4 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -101,6 +101,8 @@ def schedule(
             "derivative equals",
             "derivative max",
             "derivative min",
+            "derivative down efficiency",
+            "derivative up efficiency",
         ]
         if inflexible_device_sensors is None:
             inflexible_device_sensors = []

From ccab2eeb59803e6c603fd5cdd0a88664b4564d1d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:33:28 +0100
Subject: [PATCH 20/44] Derive flow constraints for battery scheduling, too
 (copied from Charge Point scheduler)

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/battery.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index aef80cdc2..25c5b760e 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -134,10 +134,18 @@ def schedule(
         device_constraints[0]["max"] = (soc_max - soc_at_start) * (
             timedelta(hours=1) / resolution
         )
-        device_constraints[0]["derivative min"] = (
-            sensor.get_attribute("capacity_in_mw") * -1
-        )
-        device_constraints[0]["derivative max"] = sensor.get_attribute("capacity_in_mw")
+        if sensor.get_attribute("is_strictly_non_positive"):
+            device_constraints[0]["derivative min"] = 0
+        else:
+            device_constraints[0]["derivative min"] = (
+                sensor.get_attribute("capacity_in_mw") * -1
+            )
+        if sensor.get_attribute("is_strictly_non_negative"):
+            device_constraints[0]["derivative max"] = 0
+        else:
+            device_constraints[0]["derivative max"] = sensor.get_attribute(
+                "capacity_in_mw"
+            )
 
         # Apply round-trip efficiency evenly to charging and discharging
         device_constraints[0]["derivative down efficiency"] = (

From 6344cb0c28994fb81dc4347b9b062144b6422e88 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:34:51 +0100
Subject: [PATCH 21/44] Refactor: rename BatteryScheduler to StorageScheduler

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/v1_2/implementations.py               | 4 ++--
 flexmeasures/data/models/planning/battery.py           | 2 +-
 flexmeasures/data/models/planning/tests/test_solver.py | 8 ++++----
 flexmeasures/data/services/scheduling.py               | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py
index 5ba96f9e5..26e6610d0 100644
--- a/flexmeasures/api/v1_2/implementations.py
+++ b/flexmeasures/api/v1_2/implementations.py
@@ -32,7 +32,7 @@
     parse_isodate_str,
 )
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import BatteryScheduler
+from flexmeasures.data.models.planning.battery import StorageScheduler
 from flexmeasures.data.models.planning.exceptions import (
     UnknownMarketException,
     UnknownPricesException,
@@ -106,7 +106,7 @@ def get_device_message_response(generic_asset_name_groups, duration):
                 storage_specs, sensor, start, end, resolution
             )
             try:
-                schedule = BatteryScheduler().schedule(
+                schedule = StorageScheduler().schedule(
                     sensor, start, end, resolution, storage_specs=storage_specs
                 )
             except UnknownPricesException:
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 25c5b760e..c563f6fb7 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -16,7 +16,7 @@
 )
 
 
-class BatteryScheduler(Scheduler):
+class StorageScheduler(Scheduler):
 
     __version__ = "1"
     __author__ = "Seita"
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 62eb3842e..49df49aa1 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -6,7 +6,7 @@
 import pandas as pd
 
 from flexmeasures.data.models.time_series import Sensor
-from flexmeasures.data.models.planning.battery import BatteryScheduler
+from flexmeasures.data.models.planning.battery import StorageScheduler
 from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import (
     ensure_storage_specs,
@@ -33,7 +33,7 @@ def test_battery_solver_day_1(
     storage_specs = ensure_storage_specs(
         dict(soc_at_start=soc_at_start), battery, start, end, resolution
     )
-    schedule = BatteryScheduler().schedule(
+    schedule = StorageScheduler().schedule(
         battery,
         start,
         end,
@@ -99,7 +99,7 @@ def test_battery_solver_day_2(add_battery_assets, roundtrip_efficiency: float):
         end,
         resolution,
     )
-    schedule = BatteryScheduler().schedule(
+    schedule = StorageScheduler().schedule(
         battery,
         start,
         end,
@@ -310,7 +310,7 @@ def test_building_solver_day_2(
         end,
         resolution,
     )
-    schedule = BatteryScheduler().schedule(
+    schedule = StorageScheduler().schedule(
         battery,
         start,
         end,
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 384839057..a8c541aed 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -12,7 +12,7 @@
 import timely_beliefs as tb
 
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import BatteryScheduler
+from flexmeasures.data.models.planning.battery import StorageScheduler
 from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
@@ -131,7 +131,7 @@ def make_schedule(
         scheduler_specs = sensor.attributes.get("custom-scheduler")
         scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
     elif sensor.generic_asset.generic_asset_type.name == "battery":
-        scheduler = BatteryScheduler
+        scheduler = StorageScheduler
         data_source_info["model"] = "schedule_battery"
         data_source_info["name"] = scheduler.__author__
         data_source_info["version"] = scheduler.__version__

From 7f9ecedc8ba3f0be7e8d5ba65a1cc5214568cc57 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:39:26 +0100
Subject: [PATCH 22/44] Warn for deprecation of schedule_battery and
 schedule_charging_station

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/battery.py          | 10 ++++++++++
 flexmeasures/data/models/planning/charging_station.py | 11 +++++++++++
 2 files changed, 21 insertions(+)

diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index c563f6fb7..0d8daf10e 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -181,3 +181,13 @@ def schedule(
             battery_schedule = battery_schedule.round(round_to_decimals)
 
         return battery_schedule
+
+
+def schedule_battery(*args, **kwargs):
+    import warnings
+
+    warnings.warn(
+        f"The schedule_battery method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
+        FutureWarning,
+    )
+    return StorageScheduler().schedule(*args, **kwargs)
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 0128876a4..1ea2e34b5 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -5,6 +5,7 @@
 
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.models.planning import Scheduler
+from flexmeasures.data.models.planning.battery import StorageScheduler
 from flexmeasures.data.models.planning.solver import device_scheduler
 from flexmeasures.data.models.planning.utils import (
     initialize_df,
@@ -180,3 +181,13 @@ def schedule(
             )
 
         return charging_station_schedule
+
+
+def schedule_charging_station(*args, **kwargs):
+    import warnings
+
+    warnings.warn(
+        f"The schedule_charging_station method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
+        FutureWarning,
+    )
+    return StorageScheduler().schedule(*args, **kwargs)

From 4bc593c9bcb2bde86df79c861fe4edec067fb3c9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:46:52 +0100
Subject: [PATCH 23/44] Use StorageScheduler instead of
 ChargingStationScheduler

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py         |  5 ++---
 flexmeasures/data/services/scheduling.py              | 11 +++--------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 49df49aa1..14082d62f 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -7,7 +7,6 @@
 
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.models.planning.battery import StorageScheduler
-from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import (
     ensure_storage_specs,
     initialize_series,
@@ -184,7 +183,7 @@ def test_charging_station_solver_day_2(target_soc, charging_station_name):
         end,
         resolution,
     )
-    consumption_schedule = ChargingStationScheduler().schedule(
+    consumption_schedule = StorageScheduler().schedule(
         charging_station, start, end, resolution, storage_specs=storage_specs
     )
     soc_schedule = integrate_time_series(
@@ -246,7 +245,7 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
         end,
         resolution,
     )
-    consumption_schedule = ChargingStationScheduler().schedule(
+    consumption_schedule = StorageScheduler().schedule(
         charging_station,
         start,
         end,
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index a8c541aed..97b4cfa16 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -13,7 +13,6 @@
 
 from flexmeasures.data import db
 from flexmeasures.data.models.planning.battery import StorageScheduler
-from flexmeasures.data.models.planning.charging_station import ChargingStationScheduler
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.data_sources import DataSource
@@ -130,17 +129,13 @@ def make_schedule(
     if "custom-scheduler" in sensor.attributes:
         scheduler_specs = sensor.attributes.get("custom-scheduler")
         scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
-    elif sensor.generic_asset.generic_asset_type.name == "battery":
-        scheduler = StorageScheduler
-        data_source_info["model"] = "schedule_battery"
-        data_source_info["name"] = scheduler.__author__
-        data_source_info["version"] = scheduler.__version__
     elif sensor.generic_asset.generic_asset_type.name in (
+        "battery",
         "one-way_evse",
         "two-way_evse",
     ):
-        scheduler = ChargingStationScheduler
-        data_source_info["model"] = "schedule_charging_station"
+        scheduler = StorageScheduler
+        data_source_info["model"] = scheduler.__name__
         data_source_info["name"] = scheduler.__author__
         data_source_info["version"] = scheduler.__version__
     else:

From ec40bc09834aae27dde170ce85e2f5afff7c6d1c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:48:04 +0100
Subject: [PATCH 24/44] Deprecate ChargingStationScheduler

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/charging_station.py  | 182 ------------------
 1 file changed, 182 deletions(-)

diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 1ea2e34b5..6d5455e51 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -1,186 +1,4 @@
-from typing import List, Optional, Union
-from datetime import datetime, timedelta
-
-import pandas as pd
-
-from flexmeasures.data.models.time_series import Sensor
-from flexmeasures.data.models.planning import Scheduler
 from flexmeasures.data.models.planning.battery import StorageScheduler
-from flexmeasures.data.models.planning.solver import device_scheduler
-from flexmeasures.data.models.planning.utils import (
-    initialize_df,
-    initialize_series,
-    add_tiny_price_slope,
-    get_prices,
-    get_power_values,
-    fallback_charging_policy,
-)
-
-
-class ChargingStationScheduler(Scheduler):
-
-    __version__ = "1"
-    __author__ = "Seita"
-
-    def schedule(
-        self,
-        sensor: Sensor,
-        start: datetime,
-        end: datetime,
-        resolution: timedelta,
-        storage_specs: dict,
-        consumption_price_sensor: Optional[Sensor] = None,
-        production_price_sensor: Optional[Sensor] = None,
-        inflexible_device_sensors: Optional[List[Sensor]] = None,
-        belief_time: Optional[datetime] = None,
-        round_to_decimals: Optional[int] = 6,
-    ) -> Union[pd.Series, None]:
-        """Schedule a charging station asset based directly on the latest beliefs regarding market prices within the specified time
-        window.
-        For the resulting consumption schedule, consumption is defined as positive values.
-        Todo: handle uni-directional charging by setting the "min" or "derivative min" constraint to 0
-        """
-
-        soc_at_start = storage_specs.get("soc_at_start")
-        soc_targets = storage_specs.get("soc_targets")
-        soc_min = storage_specs.get("soc_min")
-        soc_max = storage_specs.get("soc_max")
-        roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
-        prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
-
-        # Check for required Sensor attributes
-        sensor.check_required_attributes([("capacity_in_mw", (float, int))])
-
-        # Check for known prices or price forecasts, trimming planning window accordingly
-        up_deviation_prices, (start, end) = get_prices(
-            (start, end),
-            resolution,
-            beliefs_before=belief_time,
-            price_sensor=consumption_price_sensor,
-            sensor=sensor,
-            allow_trimmed_query_window=True,
-        )
-        down_deviation_prices, (start, end) = get_prices(
-            (start, end),
-            resolution,
-            beliefs_before=belief_time,
-            price_sensor=production_price_sensor,
-            sensor=sensor,
-            allow_trimmed_query_window=True,
-        )
-
-        soc_targets = soc_targets.tz_convert("UTC")
-        start = pd.Timestamp(start).tz_convert("UTC")
-        end = pd.Timestamp(end).tz_convert("UTC")
-
-        # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
-        # We penalise the future with at most 1 per thousand times the price spread.
-        if prefer_charging_sooner:
-            up_deviation_prices = add_tiny_price_slope(
-                up_deviation_prices, "event_value"
-            )
-            down_deviation_prices = add_tiny_price_slope(
-                down_deviation_prices, "event_value"
-            )
-
-        # Set up commitments to optimise for
-        commitment_quantities = [initialize_series(0, start, end, resolution)]
-
-        # Todo: convert to EUR/(deviation of commitment, which is in MW)
-        commitment_upwards_deviation_price = [
-            up_deviation_prices.loc[start : end - resolution]["event_value"]
-        ]
-        commitment_downwards_deviation_price = [
-            down_deviation_prices.loc[start : end - resolution]["event_value"]
-        ]
-
-        # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
-        columns = [
-            "equals",
-            "max",
-            "min",
-            "derivative equals",
-            "derivative max",
-            "derivative min",
-            "derivative down efficiency",
-            "derivative up efficiency",
-        ]
-        if inflexible_device_sensors is None:
-            inflexible_device_sensors = []
-        device_constraints = [
-            initialize_df(columns, start, end, resolution)
-            for i in range(1 + len(inflexible_device_sensors))
-        ]
-        for i, inflexible_sensor in enumerate(inflexible_device_sensors):
-            device_constraints[i + 1]["derivative equals"] = get_power_values(
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                sensor=inflexible_sensor,
-            )
-        device_constraints[0]["equals"] = soc_targets.shift(
-            -1, freq=resolution
-        ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
-            timedelta(hours=1) / resolution
-        )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
-        # while the "equals" constraint defines what the total stock should be at the end of a time slot,
-        # where the time slot is indexed by its starting time)
-        device_constraints[0]["min"] = (soc_min - soc_at_start) * (
-            timedelta(hours=1) / resolution
-        )
-        device_constraints[0]["max"] = (soc_max - soc_at_start) * (
-            timedelta(hours=1) / resolution
-        )
-
-        if sensor.get_attribute("is_strictly_non_positive"):
-            device_constraints[0]["derivative min"] = 0
-        else:
-            device_constraints[0]["derivative min"] = (
-                sensor.get_attribute("capacity_in_mw") * -1
-            )
-        if sensor.get_attribute("is_strictly_non_negative"):
-            device_constraints[0]["derivative max"] = 0
-        else:
-            device_constraints[0]["derivative max"] = sensor.get_attribute(
-                "capacity_in_mw"
-            )
-
-        # Apply round-trip efficiency evenly to charging and discharging
-        device_constraints[0]["derivative down efficiency"] = (
-            roundtrip_efficiency**0.5
-        )
-        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
-
-        # Set up EMS constraints
-        columns = ["derivative max", "derivative min"]
-        ems_constraints = initialize_df(columns, start, end, resolution)
-        ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
-        if ems_capacity is not None:
-            ems_constraints["derivative min"] = ems_capacity * -1
-            ems_constraints["derivative max"] = ems_capacity
-
-        ems_schedule, expected_costs, scheduler_results = device_scheduler(
-            device_constraints,
-            ems_constraints,
-            commitment_quantities,
-            commitment_downwards_deviation_price,
-            commitment_upwards_deviation_price,
-        )
-        if scheduler_results.solver.termination_condition == "infeasible":
-            # Fallback policy if the problem was unsolvable
-            charging_station_schedule = fallback_charging_policy(
-                sensor, device_constraints[0], start, end, resolution
-            )
-        else:
-            charging_station_schedule = ems_schedule[0]
-
-        # Round schedule
-        if round_to_decimals:
-            charging_station_schedule = charging_station_schedule.round(
-                round_to_decimals
-            )
-
-        return charging_station_schedule
 
 
 def schedule_charging_station(*args, **kwargs):

From 6914a4b12f6584f816e93a393d04a9a1aa6b3788 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:51:40 +0100
Subject: [PATCH 25/44] Refactor: move StorageScheduler to dedicated module

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/api/v1_2/implementations.py      |   2 +-
 flexmeasures/data/models/planning/battery.py  | 184 +-----------------
 .../data/models/planning/charging_station.py  |   2 +-
 flexmeasures/data/models/planning/storage.py  | 179 +++++++++++++++++
 .../data/models/planning/tests/test_solver.py |   2 +-
 flexmeasures/data/services/scheduling.py      |   2 +-
 6 files changed, 184 insertions(+), 187 deletions(-)
 create mode 100644 flexmeasures/data/models/planning/storage.py

diff --git a/flexmeasures/api/v1_2/implementations.py b/flexmeasures/api/v1_2/implementations.py
index 26e6610d0..1ae9f964e 100644
--- a/flexmeasures/api/v1_2/implementations.py
+++ b/flexmeasures/api/v1_2/implementations.py
@@ -32,7 +32,7 @@
     parse_isodate_str,
 )
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import StorageScheduler
+from flexmeasures.data.models.planning.storage import StorageScheduler
 from flexmeasures.data.models.planning.exceptions import (
     UnknownMarketException,
     UnknownPricesException,
diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 0d8daf10e..1812eace5 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -1,186 +1,4 @@
-from typing import List, Optional, Union
-from datetime import datetime, timedelta
-
-import pandas as pd
-
-from flexmeasures.data.models.time_series import Sensor
-from flexmeasures.data.models.planning import Scheduler
-from flexmeasures.data.models.planning.solver import device_scheduler
-from flexmeasures.data.models.planning.utils import (
-    initialize_df,
-    initialize_series,
-    add_tiny_price_slope,
-    get_prices,
-    get_power_values,
-    fallback_charging_policy,
-)
-
-
-class StorageScheduler(Scheduler):
-
-    __version__ = "1"
-    __author__ = "Seita"
-
-    def schedule(
-        self,
-        sensor: Sensor,
-        start: datetime,
-        end: datetime,
-        resolution: timedelta,
-        storage_specs: dict,
-        consumption_price_sensor: Optional[Sensor] = None,
-        production_price_sensor: Optional[Sensor] = None,
-        inflexible_device_sensors: Optional[List[Sensor]] = None,
-        belief_time: Optional[datetime] = None,
-        round_to_decimals: Optional[int] = 6,
-    ) -> Union[pd.Series, None]:
-        """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time
-        window.
-        For the resulting consumption schedule, consumption is defined as positive values.
-        """
-
-        soc_at_start = storage_specs.get("soc_at_start")
-        soc_targets = storage_specs.get("soc_targets")
-        soc_min = storage_specs.get("soc_min")
-        soc_max = storage_specs.get("soc_max")
-        roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
-        prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
-
-        # Check for required Sensor attributes
-        sensor.check_required_attributes(
-            [
-                ("capacity_in_mw", (float, int)),
-            ],
-        )
-
-        # Check for known prices or price forecasts, trimming planning window accordingly
-        up_deviation_prices, (start, end) = get_prices(
-            (start, end),
-            resolution,
-            beliefs_before=belief_time,
-            price_sensor=consumption_price_sensor,
-            sensor=sensor,
-            allow_trimmed_query_window=True,
-        )
-        down_deviation_prices, (start, end) = get_prices(
-            (start, end),
-            resolution,
-            beliefs_before=belief_time,
-            price_sensor=production_price_sensor,
-            sensor=sensor,
-            allow_trimmed_query_window=True,
-        )
-
-        start = pd.Timestamp(start).tz_convert("UTC")
-        end = pd.Timestamp(end).tz_convert("UTC")
-
-        # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
-        # We penalise the future with at most 1 per thousand times the price spread.
-        if prefer_charging_sooner:
-            up_deviation_prices = add_tiny_price_slope(
-                up_deviation_prices, "event_value"
-            )
-            down_deviation_prices = add_tiny_price_slope(
-                down_deviation_prices, "event_value"
-            )
-
-        # Set up commitments to optimise for
-        commitment_quantities = [initialize_series(0, start, end, resolution)]
-
-        # Todo: convert to EUR/(deviation of commitment, which is in MW)
-        commitment_upwards_deviation_price = [
-            up_deviation_prices.loc[start : end - resolution]["event_value"]
-        ]
-        commitment_downwards_deviation_price = [
-            down_deviation_prices.loc[start : end - resolution]["event_value"]
-        ]
-
-        # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
-        columns = [
-            "equals",
-            "max",
-            "min",
-            "derivative equals",
-            "derivative max",
-            "derivative min",
-            "derivative down efficiency",
-            "derivative up efficiency",
-        ]
-        if inflexible_device_sensors is None:
-            inflexible_device_sensors = []
-        device_constraints = [
-            initialize_df(columns, start, end, resolution)
-            for i in range(1 + len(inflexible_device_sensors))
-        ]
-        for i, inflexible_sensor in enumerate(inflexible_device_sensors):
-            device_constraints[i + 1]["derivative equals"] = get_power_values(
-                query_window=(start, end),
-                resolution=resolution,
-                beliefs_before=belief_time,
-                sensor=inflexible_sensor,
-            )
-        if soc_targets is not None:
-            soc_targets = soc_targets.tz_convert("UTC")
-            device_constraints[0]["equals"] = soc_targets.shift(
-                -1, freq=resolution
-            ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
-                timedelta(hours=1) / resolution
-            )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
-            # while the "equals" constraint defines what the total stock should be at the end of a time slot,
-            # where the time slot is indexed by its starting time)
-        device_constraints[0]["min"] = (soc_min - soc_at_start) * (
-            timedelta(hours=1) / resolution
-        )
-        device_constraints[0]["max"] = (soc_max - soc_at_start) * (
-            timedelta(hours=1) / resolution
-        )
-        if sensor.get_attribute("is_strictly_non_positive"):
-            device_constraints[0]["derivative min"] = 0
-        else:
-            device_constraints[0]["derivative min"] = (
-                sensor.get_attribute("capacity_in_mw") * -1
-            )
-        if sensor.get_attribute("is_strictly_non_negative"):
-            device_constraints[0]["derivative max"] = 0
-        else:
-            device_constraints[0]["derivative max"] = sensor.get_attribute(
-                "capacity_in_mw"
-            )
-
-        # Apply round-trip efficiency evenly to charging and discharging
-        device_constraints[0]["derivative down efficiency"] = (
-            roundtrip_efficiency**0.5
-        )
-        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
-
-        # Set up EMS constraints
-        columns = ["derivative max", "derivative min"]
-        ems_constraints = initialize_df(columns, start, end, resolution)
-        ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
-        if ems_capacity is not None:
-            ems_constraints["derivative min"] = ems_capacity * -1
-            ems_constraints["derivative max"] = ems_capacity
-
-        ems_schedule, expected_costs, scheduler_results = device_scheduler(
-            device_constraints,
-            ems_constraints,
-            commitment_quantities,
-            commitment_downwards_deviation_price,
-            commitment_upwards_deviation_price,
-        )
-        if scheduler_results.solver.termination_condition == "infeasible":
-            # Fallback policy if the problem was unsolvable
-            battery_schedule = fallback_charging_policy(
-                sensor, device_constraints[0], start, end, resolution
-            )
-        else:
-            battery_schedule = ems_schedule[0]
-
-        # Round schedule
-        if round_to_decimals:
-            battery_schedule = battery_schedule.round(round_to_decimals)
-
-        return battery_schedule
+from flexmeasures.data.models.planning.storage import StorageScheduler
 
 
 def schedule_battery(*args, **kwargs):
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 6d5455e51..803aef172 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -1,4 +1,4 @@
-from flexmeasures.data.models.planning.battery import StorageScheduler
+from flexmeasures.data.models.planning.storage import StorageScheduler
 
 
 def schedule_charging_station(*args, **kwargs):
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
new file mode 100644
index 000000000..bfd7c8523
--- /dev/null
+++ b/flexmeasures/data/models/planning/storage.py
@@ -0,0 +1,179 @@
+from datetime import datetime, timedelta
+from typing import Optional, List, Union
+
+import pandas as pd
+
+from flexmeasures import Sensor
+from flexmeasures.data.models.planning import Scheduler
+from flexmeasures.data.models.planning.solver import device_scheduler
+from flexmeasures.data.models.planning.utils import (
+    get_prices,
+    add_tiny_price_slope,
+    initialize_series,
+    initialize_df,
+    get_power_values,
+    fallback_charging_policy,
+)
+
+
+class StorageScheduler(Scheduler):
+
+    __version__ = "1"
+    __author__ = "Seita"
+
+    def schedule(
+        self,
+        sensor: Sensor,
+        start: datetime,
+        end: datetime,
+        resolution: timedelta,
+        storage_specs: dict,
+        consumption_price_sensor: Optional[Sensor] = None,
+        production_price_sensor: Optional[Sensor] = None,
+        inflexible_device_sensors: Optional[List[Sensor]] = None,
+        belief_time: Optional[datetime] = None,
+        round_to_decimals: Optional[int] = 6,
+    ) -> Union[pd.Series, None]:
+        """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time
+        window.
+        For the resulting consumption schedule, consumption is defined as positive values.
+        """
+
+        soc_at_start = storage_specs.get("soc_at_start")
+        soc_targets = storage_specs.get("soc_targets")
+        soc_min = storage_specs.get("soc_min")
+        soc_max = storage_specs.get("soc_max")
+        roundtrip_efficiency = storage_specs.get("roundtrip_efficiency")
+        prefer_charging_sooner = storage_specs.get("prefer_charging_sooner", True)
+
+        # Check for required Sensor attributes
+        sensor.check_required_attributes([("capacity_in_mw", (float, int))])
+
+        # Check for known prices or price forecasts, trimming planning window accordingly
+        up_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
+            beliefs_before=belief_time,
+            price_sensor=consumption_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
+        )
+        down_deviation_prices, (start, end) = get_prices(
+            (start, end),
+            resolution,
+            beliefs_before=belief_time,
+            price_sensor=production_price_sensor,
+            sensor=sensor,
+            allow_trimmed_query_window=True,
+        )
+
+        start = pd.Timestamp(start).tz_convert("UTC")
+        end = pd.Timestamp(end).tz_convert("UTC")
+
+        # Add tiny price slope to prefer charging now rather than later, and discharging later rather than now.
+        # We penalise the future with at most 1 per thousand times the price spread.
+        if prefer_charging_sooner:
+            up_deviation_prices = add_tiny_price_slope(
+                up_deviation_prices, "event_value"
+            )
+            down_deviation_prices = add_tiny_price_slope(
+                down_deviation_prices, "event_value"
+            )
+
+        # Set up commitments to optimise for
+        commitment_quantities = [initialize_series(0, start, end, resolution)]
+
+        # Todo: convert to EUR/(deviation of commitment, which is in MW)
+        commitment_upwards_deviation_price = [
+            up_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+        commitment_downwards_deviation_price = [
+            down_deviation_prices.loc[start : end - resolution]["event_value"]
+        ]
+
+        # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n).
+        columns = [
+            "equals",
+            "max",
+            "min",
+            "derivative equals",
+            "derivative max",
+            "derivative min",
+            "derivative down efficiency",
+            "derivative up efficiency",
+        ]
+        if inflexible_device_sensors is None:
+            inflexible_device_sensors = []
+        device_constraints = [
+            initialize_df(columns, start, end, resolution)
+            for i in range(1 + len(inflexible_device_sensors))
+        ]
+        for i, inflexible_sensor in enumerate(inflexible_device_sensors):
+            device_constraints[i + 1]["derivative equals"] = get_power_values(
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+                sensor=inflexible_sensor,
+            )
+        if soc_targets is not None:
+            soc_targets = soc_targets.tz_convert("UTC")
+            device_constraints[0]["equals"] = soc_targets.shift(
+                -1, freq=resolution
+            ).values * (timedelta(hours=1) / resolution) - soc_at_start * (
+                timedelta(hours=1) / resolution
+            )  # shift "equals" constraint for target SOC by one resolution (the target defines a state at a certain time,
+            # while the "equals" constraint defines what the total stock should be at the end of a time slot,
+            # where the time slot is indexed by its starting time)
+        device_constraints[0]["min"] = (soc_min - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
+        device_constraints[0]["max"] = (soc_max - soc_at_start) * (
+            timedelta(hours=1) / resolution
+        )
+        if sensor.get_attribute("is_strictly_non_positive"):
+            device_constraints[0]["derivative min"] = 0
+        else:
+            device_constraints[0]["derivative min"] = (
+                sensor.get_attribute("capacity_in_mw") * -1
+            )
+        if sensor.get_attribute("is_strictly_non_negative"):
+            device_constraints[0]["derivative max"] = 0
+        else:
+            device_constraints[0]["derivative max"] = sensor.get_attribute(
+                "capacity_in_mw"
+            )
+
+        # Apply round-trip efficiency evenly to charging and discharging
+        device_constraints[0]["derivative down efficiency"] = (
+            roundtrip_efficiency**0.5
+        )
+        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
+
+        # Set up EMS constraints
+        columns = ["derivative max", "derivative min"]
+        ems_constraints = initialize_df(columns, start, end, resolution)
+        ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw")
+        if ems_capacity is not None:
+            ems_constraints["derivative min"] = ems_capacity * -1
+            ems_constraints["derivative max"] = ems_capacity
+
+        ems_schedule, expected_costs, scheduler_results = device_scheduler(
+            device_constraints,
+            ems_constraints,
+            commitment_quantities,
+            commitment_downwards_deviation_price,
+            commitment_upwards_deviation_price,
+        )
+        if scheduler_results.solver.termination_condition == "infeasible":
+            # Fallback policy if the problem was unsolvable
+            battery_schedule = fallback_charging_policy(
+                sensor, device_constraints[0], start, end, resolution
+            )
+        else:
+            battery_schedule = ems_schedule[0]
+
+        # Round schedule
+        if round_to_decimals:
+            battery_schedule = battery_schedule.round(round_to_decimals)
+
+        return battery_schedule
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 14082d62f..ef2b9711f 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -6,7 +6,7 @@
 import pandas as pd
 
 from flexmeasures.data.models.time_series import Sensor
-from flexmeasures.data.models.planning.battery import StorageScheduler
+from flexmeasures.data.models.planning.storage import StorageScheduler
 from flexmeasures.data.models.planning.utils import (
     ensure_storage_specs,
     initialize_series,
diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 97b4cfa16..0eb78bf1e 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -12,7 +12,7 @@
 import timely_beliefs as tb
 
 from flexmeasures.data import db
-from flexmeasures.data.models.planning.battery import StorageScheduler
+from flexmeasures.data.models.planning.storage import StorageScheduler
 from flexmeasures.data.models.planning.utils import ensure_storage_specs
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.data_sources import DataSource

From 5e6bd4e69f44ea349dad6a469c561109a85bf3b9 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 15:52:34 +0100
Subject: [PATCH 26/44] Update docstring

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index bfd7c8523..3958ac6da 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -34,8 +34,7 @@ def schedule(
         belief_time: Optional[datetime] = None,
         round_to_decimals: Optional[int] = 6,
     ) -> Union[pd.Series, None]:
-        """Schedule a battery asset based directly on the latest beliefs regarding market prices within the specified time
-        window.
+        """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window.
         For the resulting consumption schedule, consumption is defined as positive values.
         """
 

From beb277091c1f32c2f4ef38db1362a499d8a6d954 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 17:21:44 +0100
Subject: [PATCH 27/44] fix test

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/services/scheduling.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py
index 0eb78bf1e..510c59b4a 100644
--- a/flexmeasures/data/services/scheduling.py
+++ b/flexmeasures/data/services/scheduling.py
@@ -291,12 +291,7 @@ def get_data_source_for_job(
         if data_source_info and "id" in data_source_info:
             return DataSource.query.get(data_source_info["id"])
     if data_source_info is None and sensor:
-        data_source_info = dict(
-            name="Seita",
-            model="schedule_battery"
-            if sensor.generic_asset.generic_asset_type.name == "battery"
-            else "schedule_charging_station",
-        )
+        data_source_info = dict(name="Seita", model="StorageScheduler")
         # TODO: change to raise later (v0.13) - all scheduling jobs now get full info
         current_app.logger.warning(
             "Looking up scheduling data without knowing full data_source_info (version). This is deprecated soon. Please specify a job id as event or switch to API v3."

From 3bf1b973232b641ccc13ca51dab5760a14f90ae7 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 19:33:39 +0100
Subject: [PATCH 28/44] flake8

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/battery.py          | 2 +-
 flexmeasures/data/models/planning/charging_station.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/planning/battery.py b/flexmeasures/data/models/planning/battery.py
index 1812eace5..c8004792a 100644
--- a/flexmeasures/data/models/planning/battery.py
+++ b/flexmeasures/data/models/planning/battery.py
@@ -5,7 +5,7 @@ def schedule_battery(*args, **kwargs):
     import warnings
 
     warnings.warn(
-        f"The schedule_battery method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
+        "The schedule_battery method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
         FutureWarning,
     )
     return StorageScheduler().schedule(*args, **kwargs)
diff --git a/flexmeasures/data/models/planning/charging_station.py b/flexmeasures/data/models/planning/charging_station.py
index 803aef172..32ef204cf 100644
--- a/flexmeasures/data/models/planning/charging_station.py
+++ b/flexmeasures/data/models/planning/charging_station.py
@@ -5,7 +5,7 @@ def schedule_charging_station(*args, **kwargs):
     import warnings
 
     warnings.warn(
-        f"The schedule_charging_station method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
+        "The schedule_charging_station method is deprecated and will be removed from flexmeasures in a future version. Replace with StorageScheduler().schedule to suppress this warning.",
         FutureWarning,
     )
     return StorageScheduler().schedule(*args, **kwargs)

From a9899a282ab27f56081fa549d9e427d8ea1e3772 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 19:44:43 +0100
Subject: [PATCH 29/44] Lose the v in version strings; prefer versions showing
 up as 'version: 3' over 'version: v3'.

Even though Scheduler versioning does not necessarily need to follow semantic versioning (see discussion here: https://github.com/semver/semver/issues/235), the v is still redundant.

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/plugin/customisation.rst          | 2 +-
 flexmeasures/data/tests/dummy_scheduler.py      | 2 +-
 flexmeasures/data/tests/test_scheduling_jobs.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/documentation/plugin/customisation.rst b/documentation/plugin/customisation.rst
index c3a03ad03..d13401d6b 100644
--- a/documentation/plugin/customisation.rst
+++ b/documentation/plugin/customisation.rst
@@ -30,7 +30,7 @@ The following minimal example gives you an idea of some meta information you can
     class DummyScheduler(Scheduler):
 
         __author__ = "My Company"
-        __version__ = "v2"
+        __version__ = "2"
 
         def schedule(
             self,
diff --git a/flexmeasures/data/tests/dummy_scheduler.py b/flexmeasures/data/tests/dummy_scheduler.py
index 19908dee3..7a97c2b46 100644
--- a/flexmeasures/data/tests/dummy_scheduler.py
+++ b/flexmeasures/data/tests/dummy_scheduler.py
@@ -8,7 +8,7 @@
 class DummyScheduler(Scheduler):
 
     __author__ = "Test Organization"
-    __version__ = "v3"
+    __version__ = "3"
 
     def schedule(
         self,
diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py
index 676eee92d..357b3709a 100644
--- a/flexmeasures/data/tests/test_scheduling_jobs.py
+++ b/flexmeasures/data/tests/test_scheduling_jobs.py
@@ -80,7 +80,7 @@ def test_loading_custom_scheduler(is_path: bool):
     scheduler_specs["module"] = make_module_descr(is_path)
     custom_scheduler, data_source_info = load_custom_scheduler(scheduler_specs)
     assert data_source_info["name"] == "Test Organization"
-    assert data_source_info["version"] == "v3"
+    assert data_source_info["version"] == "3"
     assert data_source_info["model"] == "DummyScheduler"
     assert custom_scheduler.__name__ == "DummyScheduler"
     assert "Just a dummy scheduler" in custom_scheduler.schedule.__doc__

From ad22c3541729859d52cdb956740f1d3ab5059642 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 12:48:07 +0100
Subject: [PATCH 30/44] Refactor: rename module

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/{solver.py => linear_optimization.py}  | 0
 flexmeasures/data/models/planning/storage.py                    | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename flexmeasures/data/models/planning/{solver.py => linear_optimization.py} (100%)

diff --git a/flexmeasures/data/models/planning/solver.py b/flexmeasures/data/models/planning/linear_optimization.py
similarity index 100%
rename from flexmeasures/data/models/planning/solver.py
rename to flexmeasures/data/models/planning/linear_optimization.py
diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index 3958ac6da..c03f9742e 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -5,7 +5,7 @@
 
 from flexmeasures import Sensor
 from flexmeasures.data.models.planning import Scheduler
-from flexmeasures.data.models.planning.solver import device_scheduler
+from flexmeasures.data.models.planning.linear_optimization import device_scheduler
 from flexmeasures.data.models.planning.utils import (
     get_prices,
     add_tiny_price_slope,

From 4efe8839b38565d684c90520c2b03a09218f5a84 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 12:51:39 +0100
Subject: [PATCH 31/44] Deal with empty SoC targets

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py
index c03f9742e..ddc5ce793 100644
--- a/flexmeasures/data/models/planning/storage.py
+++ b/flexmeasures/data/models/planning/storage.py
@@ -114,7 +114,7 @@ def schedule(
                 beliefs_before=belief_time,
                 sensor=inflexible_sensor,
             )
-        if soc_targets is not None:
+        if soc_targets is not None and not soc_targets.empty:
             soc_targets = soc_targets.tz_convert("UTC")
             device_constraints[0]["equals"] = soc_targets.shift(
                 -1, freq=resolution

From 09cf7002a7330029350d1c868c1351ca573fc057 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 9 Oct 2022 11:34:15 +0200
Subject: [PATCH 32/44] Stop wrapping DataFrame representations in logging

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/utils/config_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flexmeasures/utils/config_utils.py b/flexmeasures/utils/config_utils.py
index bc7cb5fbe..519203d77 100644
--- a/flexmeasures/utils/config_utils.py
+++ b/flexmeasures/utils/config_utils.py
@@ -8,6 +8,7 @@
 
 from flask import Flask
 from inflection import camelize
+import pandas as pd
 
 from flexmeasures.utils.config_defaults import (
     Config as DefaultConfig,
@@ -45,6 +46,7 @@
 
 def configure_logging():
     """Configure and register logging"""
+    pd.options.display.expand_frame_repr = False  # Don't wrap DataFrame representations
     loggingDictConfig(flexmeasures_logging_config)
 
 

From 5a3845dd4cc3fd83b085d944a3e48cf80a4d3797 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Sun, 9 Oct 2022 11:17:39 +0200
Subject: [PATCH 33/44] Log warning instead of raising
 UnknownForecastException, and assume zero power values for missing values

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index 6b7f56c96..d4442583d 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -10,7 +10,6 @@
 
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
 from flexmeasures.data.models.planning.exceptions import (
-    UnknownForecastException,
     UnknownMarketException,
     UnknownPricesException,
 )
@@ -238,11 +237,13 @@ def get_power_values(
         one_deterministic_belief_per_event=True,
     )  # consumption is negative, production is positive
     df = simplify_index(bdf)
+    df = df.reindex(initialize_index(query_window[0], query_window[1], resolution))
     nan_values = df.isnull().values
     if nan_values.any() or df.empty:
-        raise UnknownForecastException(
-            f"Forecasts unknown for planning window. (sensor {sensor.id})"
+        current_app.logger.warning(
+            f"Assuming zero power values for (partially) unknown power values for planning window. (sensor {sensor.id})"
         )
+        df = df.fillna(0)
     if sensor.get_attribute(
         "consumption_is_positive", False
     ):  # FlexMeasures default is to store consumption as negative power values

From 4713de1bae96914fadd3a52b5697f9785e47ba12 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 10 Nov 2022 09:20:22 +0100
Subject: [PATCH 34/44] Workaround for GH #484

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/time_series.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py
index 958e61201..638614853 100644
--- a/flexmeasures/data/models/time_series.py
+++ b/flexmeasures/data/models/time_series.py
@@ -649,11 +649,28 @@ def search(
 
         bdf_dict = {}
         for sensor in sensors:
+            # Workaround (1st half) for https://github.com/FlexMeasures/flexmeasures/issues/484
+            if event_starts_after is not None:
+                _event_starts_after = (
+                        event_starts_after
+                        - sensor.event_resolution
+                        + timedelta(milliseconds=1)
+                )
+            else:
+                _event_starts_after = event_starts_after
+            if event_ends_before is not None:
+                _event_ends_before = (
+                        event_ends_before
+                        + sensor.event_resolution
+                        - timedelta(milliseconds=1)
+                )
+            else:
+                _event_ends_before = event_ends_before
             bdf = cls.search_session(
                 session=db.session,
                 sensor=sensor,
-                event_starts_after=event_starts_after,
-                event_ends_before=event_ends_before,
+                event_starts_after=_event_starts_after,
+                event_ends_before=_event_ends_before,
                 beliefs_after=beliefs_after,
                 beliefs_before=beliefs_before,
                 horizons_at_least=horizons_at_least,
@@ -690,6 +707,9 @@ def search(
                 bdf = bdf.resample_events(
                     resolution, keep_only_most_recent_belief=most_recent_beliefs_only
                 )
+                # Workaround (2nd half) for https://github.com/FlexMeasures/flexmeasures/issues/484
+                bdf = bdf[bdf.event_starts >= event_starts_after]
+                bdf = bdf[bdf.event_ends <= event_ends_before]
             bdf_dict[bdf.sensor.name] = bdf
 
         if sum_multiple:

From 0780d53554a252e9a0b72c2b64d41b95203d4c3a Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 15:32:47 +0100
Subject: [PATCH 35/44] Test maximizing self-consumption

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/conftest.py    |  56 ++++++++++
 .../data/models/planning/tests/test_solver.py | 103 +++++++++++++++++-
 2 files changed, 156 insertions(+), 3 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py
index e59db120f..ebfe12379 100644
--- a/flexmeasures/data/models/planning/tests/conftest.py
+++ b/flexmeasures/data/models/planning/tests/conftest.py
@@ -3,6 +3,7 @@
 from datetime import timedelta
 import pytest
 
+from timely_beliefs.sensors.func_store.knowledge_horizons import at_date
 import pandas as pd
 
 from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
@@ -18,6 +19,61 @@ def setup_planning_test_data(db, add_market_prices, add_charging_station_assets)
     print("Setting up data for planning tests on %s" % db.engine)
 
 
+@pytest.fixture(scope="module")
+def create_test_tariffs(db, setup_accounts, setup_sources) -> Dict[str, Sensor]:
+    """Create a fixed consumption tariff and a fixed feed-in tariff that is lower."""
+
+    market_type = GenericAssetType(
+        name="tariff market",
+    )
+    db.session.add(market_type)
+    contract = GenericAsset(
+        name="supply contract",
+        generic_asset_type=market_type,
+        owner=setup_accounts["Supplier"],
+    )
+    db.session.add(contract)
+    consumption_price_sensor = Sensor(
+        name="fixed consumption tariff",
+        generic_asset=contract,
+        event_resolution=timedelta(hours=24 * 365),
+        unit="EUR/MWh",
+        knowledge_horizon=(at_date, {"knowledge_time": "2014-11-01T00:00+01:00"}),
+    )
+    db.session.add(consumption_price_sensor)
+    production_price_sensor = Sensor(
+        name="fixed feed-in tariff",
+        generic_asset=contract,
+        event_resolution=timedelta(hours=24 * 365),
+        unit="EUR/MWh",
+        knowledge_horizon=(at_date, {"knowledge_time": "2014-11-01T00:00+01:00"}),
+    )
+    db.session.add(production_price_sensor)
+
+    # Add prices
+    consumption_price = TimedBelief(
+        event_start="2015-01-01T00:00+01:00",
+        belief_time="2014-11-01T00:00+01:00",  # publication date
+        event_value=300 * 1.21,
+        source=setup_sources["Seita"],
+        sensor=consumption_price_sensor,
+    )
+    db.session.add(consumption_price)
+    production_price = TimedBelief(
+        event_start="2015-01-01T00:00+01:00",
+        belief_time="2014-11-01T00:00+01:00",  # publication date
+        event_value=300,
+        source=setup_sources["Seita"],
+        sensor=production_price_sensor,
+    )
+    db.session.add(production_price)
+    db.session.flush()  # make sure that prices are assigned to price sensors
+    return {
+        "consumption_price_sensor": consumption_price_sensor,
+        "production_price_sensor": production_price_sensor,
+    }
+
+
 @pytest.fixture(scope="module")
 def building(db, setup_accounts, setup_markets) -> GenericAsset:
     """
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index ef2b9711f..c1a8b3b2a 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -283,9 +283,9 @@ def test_building_solver_day_2(
     """Check battery scheduling results within the context of a building with PV, for day 2,
     which is set up with 8 expensive, then 8 cheap, then again 8 expensive hours.
     We expect the scheduler to:
-    - completely discharge within the first 8 hours
-    - completely charge within the next 8 hours
-    - completely discharge within the last 8 hours
+    - completely discharge within the first 8 hours (with high prices)
+    - completely charge within the next 8 hours (with low prices)
+    - completely discharge within the last 8 hours (with high prices)
     """
     epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none()
     battery = flexible_devices["battery power sensor"]
@@ -364,3 +364,100 @@ def test_building_solver_day_2(
     assert soc_schedule.loc[start + timedelta(hours=16)] == min(
         soc_max, battery.get_attribute("max_soc_in_mwh")
     )  # Buy what you can to sell later
+
+
+def test_building_solver_with_fixed_tariffs_day_2(
+    db,
+    add_battery_assets,
+    create_test_tariffs,
+    add_inflexible_device_forecasts,
+    inflexible_devices,
+    flexible_devices,
+):
+    """Check battery scheduling results within the context of a building with PV, for day 2,
+    against a fixed consumption tariff and a fixed feed-in tariff that is lower.
+    We expect the scheduler to:
+    - completely discharge within the first 8 hours (with net consumption)
+    - completely charge within the next 8 hours (with net production)
+    - completely discharge within the last 8 hours (with net consumption)
+    """
+    consumption_price_sensor = create_test_tariffs["consumption_price_sensor"]
+    production_price_sensor = create_test_tariffs["production_price_sensor"]
+    battery = flexible_devices["battery power sensor"]
+    building = battery.generic_asset
+
+    tz = pytz.timezone("Europe/Amsterdam")
+    start = tz.localize(datetime(2015, 1, 2))
+    end = tz.localize(datetime(2015, 1, 3))
+    resolution = timedelta(minutes=15)
+    soc_at_start = 2.5
+    soc_min = 0.5
+    soc_max = 4.5
+    storage_specs = ensure_storage_specs(
+        dict(
+            soc_at_start=soc_at_start,
+            soc_min=soc_min,
+            soc_max=soc_max,
+        ),
+        battery,
+        start,
+        end,
+        resolution,
+    )
+    schedule = StorageScheduler().schedule(
+        battery,
+        start,
+        end,
+        resolution,
+        storage_specs=storage_specs,
+        consumption_price_sensor=consumption_price_sensor,
+        production_price_sensor=production_price_sensor,
+        inflexible_device_sensors=inflexible_devices.values(),
+    )
+    soc_schedule = integrate_time_series(schedule, soc_at_start, decimal_precision=6)
+
+    with pd.option_context("display.max_rows", None, "display.max_columns", 3):
+        print(soc_schedule)
+
+    # Check if constraints were met
+    capacity = pd.DataFrame(
+        data=np.sum(np.array(list(add_inflexible_device_forecasts.values())), axis=0),
+        columns=["inflexible"],
+    ).tail(
+        -4 * 24
+    )  # remove first 96 quarter-hours (the schedule is about the 2nd day)
+    capacity["max"] = building.get_attribute("capacity_in_mw")
+    capacity["min"] = -building.get_attribute("capacity_in_mw")
+    capacity["production headroom"] = capacity["max"] - capacity["inflexible"]
+    capacity["consumption headroom"] = capacity["inflexible"] - capacity["min"]
+    capacity["battery production headroom"] = capacity["production headroom"].clip(
+        upper=battery.get_attribute("capacity_in_mw")
+    )
+    capacity["battery consumption headroom"] = capacity["consumption headroom"].clip(
+        upper=battery.get_attribute("capacity_in_mw")
+    )
+    capacity[
+        "schedule"
+    ] = schedule.values  # consumption is positive, production is negative
+    with pd.option_context(
+        "display.max_rows", None, "display.max_columns", None, "display.width", 2000
+    ):
+        print(capacity)
+    assert (capacity["schedule"] >= -capacity["battery production headroom"]).all()
+    assert (capacity["schedule"] <= capacity["battery consumption headroom"]).all()
+
+    for soc in soc_schedule.values:
+        assert soc >= max(soc_min, battery.get_attribute("min_soc_in_mwh"))
+        assert soc <= battery.get_attribute("max_soc_in_mwh")
+
+    # Check whether the resulting soc schedule follows our expectations for 8 net-consumption, 8 net-production and 8 net-consumption hours
+    assert soc_schedule.iloc[-1] == max(
+        soc_min, battery.get_attribute("min_soc_in_mwh")
+    )  # The battery discharged as far as it could during the last 8 net-consumption hours
+
+    assert soc_schedule.loc[start + timedelta(hours=8)] == max(
+        soc_min, battery.get_attribute("min_soc_in_mwh")
+    )  # The battery discharged as far as it could during the first 8 net-consumption hours
+    assert soc_schedule.loc[start + timedelta(hours=16)] == min(
+        soc_max, battery.get_attribute("max_soc_in_mwh")
+    )  # battery charged with PV power as far as it could during the middle 8 net-production hours

From 51e5809ed9e105ed3472264f49d311a38cac2000 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 15:46:57 +0100
Subject: [PATCH 36/44] Refactor: a single parameterized test instead of 2
 tests with a lot of duplicate code

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 151 +++++-------------
 flexmeasures/data/models/planning/utils.py    |  10 ++
 2 files changed, 54 insertions(+), 107 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index c1a8b3b2a..02bcebb4d 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -272,25 +272,49 @@ def test_fallback_to_unsolvable_problem(target_soc, charging_station_name):
     )
 
 
+@pytest.mark.parametrize(
+    "market_scenario",
+    [
+        "dynamic contract",
+        "fixed contract",
+    ],
+)
 def test_building_solver_day_2(
     db,
     add_battery_assets,
     add_market_prices,
+    create_test_tariffs,
     add_inflexible_device_forecasts,
     inflexible_devices,
     flexible_devices,
+    market_scenario: str,
 ):
-    """Check battery scheduling results within the context of a building with PV, for day 2,
-    which is set up with 8 expensive, then 8 cheap, then again 8 expensive hours.
-    We expect the scheduler to:
-    - completely discharge within the first 8 hours (with high prices)
-    - completely charge within the next 8 hours (with low prices)
-    - completely discharge within the last 8 hours (with high prices)
+    """Check battery scheduling results within the context of a building with PV, for day 2, against the following market scenarios:
+    1) a dynamic tariff with equal consumption and feed-in tariffs, that is set up with 8 expensive, then 8 cheap, then again 8 expensive hours.
+    2) a fixed consumption tariff and a fixed feed-in tariff that is lower.
+    In the test data:
+    - Hours with net production coincide with low dynamic market prices.
+    - Hours with net consumption coincide with high dynamic market prices.
+    In either scenario, we expect the scheduler to:
+    - completely discharge within the first 8 hours (either due to 1) high prices, or 2) net consumption)
+    - completely charge within the next 8 hours (either due to 1) low prices, or 2) net production)
+    - completely discharge within the last 8 hours (either due to 1) high prices, or 2) net consumption)
     """
-    epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none()
     battery = flexible_devices["battery power sensor"]
     building = battery.generic_asset
-    assert battery.get_attribute("market_id") == epex_da.id
+    if market_scenario == "dynamic contract":
+        consumption_price_sensor = Sensor.query.filter(
+            Sensor.name == "epex_da"
+        ).one_or_none()
+        production_price_sensor = consumption_price_sensor
+        assert battery.get_attribute("market_id") == consumption_price_sensor.id
+    elif market_scenario == "fixed contract":
+        consumption_price_sensor = create_test_tariffs["consumption_price_sensor"]
+        production_price_sensor = create_test_tariffs["production_price_sensor"]
+    else:
+        raise NotImplementedError(
+            f"Missing test case for market conditions '{market_scenario}'"
+        )
     tz = pytz.timezone("Europe/Amsterdam")
     start = tz.localize(datetime(2015, 1, 2))
     end = tz.localize(datetime(2015, 1, 3))
@@ -353,111 +377,24 @@ def test_building_solver_day_2(
         assert soc >= max(soc_min, battery.get_attribute("min_soc_in_mwh"))
         assert soc <= battery.get_attribute("max_soc_in_mwh")
 
-    # Check whether the resulting soc schedule follows our expectations for 8 expensive, 8 cheap and 8 expensive hours
+    # Check whether the resulting soc schedule follows our expectations for:
+    # 1) 8 expensive, 8 cheap and 8 expensive hours
+    # 2) 8 net-consumption, 8 net-production and 8 net-consumption hours
+
+    # 1) The battery sold out at the end of its planning horizon
+    # 2) The battery discharged as far as it could during the last 8 net-consumption hours
     assert soc_schedule.iloc[-1] == max(
         soc_min, battery.get_attribute("min_soc_in_mwh")
-    )  # Battery sold out at the end of its planning horizon
+    )
 
+    # 1) Sell what you begin with
+    # 2) The battery discharged as far as it could during the first 8 net-consumption hours
     assert soc_schedule.loc[start + timedelta(hours=8)] == max(
         soc_min, battery.get_attribute("min_soc_in_mwh")
-    )  # Sell what you begin with
-    assert soc_schedule.loc[start + timedelta(hours=16)] == min(
-        soc_max, battery.get_attribute("max_soc_in_mwh")
-    )  # Buy what you can to sell later
-
-
-def test_building_solver_with_fixed_tariffs_day_2(
-    db,
-    add_battery_assets,
-    create_test_tariffs,
-    add_inflexible_device_forecasts,
-    inflexible_devices,
-    flexible_devices,
-):
-    """Check battery scheduling results within the context of a building with PV, for day 2,
-    against a fixed consumption tariff and a fixed feed-in tariff that is lower.
-    We expect the scheduler to:
-    - completely discharge within the first 8 hours (with net consumption)
-    - completely charge within the next 8 hours (with net production)
-    - completely discharge within the last 8 hours (with net consumption)
-    """
-    consumption_price_sensor = create_test_tariffs["consumption_price_sensor"]
-    production_price_sensor = create_test_tariffs["production_price_sensor"]
-    battery = flexible_devices["battery power sensor"]
-    building = battery.generic_asset
-
-    tz = pytz.timezone("Europe/Amsterdam")
-    start = tz.localize(datetime(2015, 1, 2))
-    end = tz.localize(datetime(2015, 1, 3))
-    resolution = timedelta(minutes=15)
-    soc_at_start = 2.5
-    soc_min = 0.5
-    soc_max = 4.5
-    storage_specs = ensure_storage_specs(
-        dict(
-            soc_at_start=soc_at_start,
-            soc_min=soc_min,
-            soc_max=soc_max,
-        ),
-        battery,
-        start,
-        end,
-        resolution,
-    )
-    schedule = StorageScheduler().schedule(
-        battery,
-        start,
-        end,
-        resolution,
-        storage_specs=storage_specs,
-        consumption_price_sensor=consumption_price_sensor,
-        production_price_sensor=production_price_sensor,
-        inflexible_device_sensors=inflexible_devices.values(),
     )
-    soc_schedule = integrate_time_series(schedule, soc_at_start, decimal_precision=6)
-
-    with pd.option_context("display.max_rows", None, "display.max_columns", 3):
-        print(soc_schedule)
 
-    # Check if constraints were met
-    capacity = pd.DataFrame(
-        data=np.sum(np.array(list(add_inflexible_device_forecasts.values())), axis=0),
-        columns=["inflexible"],
-    ).tail(
-        -4 * 24
-    )  # remove first 96 quarter-hours (the schedule is about the 2nd day)
-    capacity["max"] = building.get_attribute("capacity_in_mw")
-    capacity["min"] = -building.get_attribute("capacity_in_mw")
-    capacity["production headroom"] = capacity["max"] - capacity["inflexible"]
-    capacity["consumption headroom"] = capacity["inflexible"] - capacity["min"]
-    capacity["battery production headroom"] = capacity["production headroom"].clip(
-        upper=battery.get_attribute("capacity_in_mw")
-    )
-    capacity["battery consumption headroom"] = capacity["consumption headroom"].clip(
-        upper=battery.get_attribute("capacity_in_mw")
-    )
-    capacity[
-        "schedule"
-    ] = schedule.values  # consumption is positive, production is negative
-    with pd.option_context(
-        "display.max_rows", None, "display.max_columns", None, "display.width", 2000
-    ):
-        print(capacity)
-    assert (capacity["schedule"] >= -capacity["battery production headroom"]).all()
-    assert (capacity["schedule"] <= capacity["battery consumption headroom"]).all()
-
-    for soc in soc_schedule.values:
-        assert soc >= max(soc_min, battery.get_attribute("min_soc_in_mwh"))
-        assert soc <= battery.get_attribute("max_soc_in_mwh")
-
-    # Check whether the resulting soc schedule follows our expectations for 8 net-consumption, 8 net-production and 8 net-consumption hours
-    assert soc_schedule.iloc[-1] == max(
-        soc_min, battery.get_attribute("min_soc_in_mwh")
-    )  # The battery discharged as far as it could during the last 8 net-consumption hours
-
-    assert soc_schedule.loc[start + timedelta(hours=8)] == max(
-        soc_min, battery.get_attribute("min_soc_in_mwh")
-    )  # The battery discharged as far as it could during the first 8 net-consumption hours
+    # 1) Buy what you can to sell later
+    # 2) The battery charged with PV power as far as it could during the middle 8 net-production hours
     assert soc_schedule.loc[start + timedelta(hours=16)] == min(
         soc_max, battery.get_attribute("max_soc_in_mwh")
-    )  # battery charged with PV power as far as it could during the middle 8 net-production hours
+    )
diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index d4442583d..b7a4c6770 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -182,6 +182,11 @@ def get_prices(
         most_recent_beliefs_only=True,
         one_deterministic_belief_per_event=True,
     )
+    with pd.option_context(
+            "display.max_rows", None, "display.max_columns", None, "display.width", 2000
+    ):
+        print(price_bdf)
+        print(price_bdf.sensor)
     price_df = simplify_index(price_bdf)
     nan_prices = price_df.isnull().values
     if nan_prices.all() or price_df.empty:
@@ -236,6 +241,11 @@ def get_power_values(
         most_recent_beliefs_only=True,
         one_deterministic_belief_per_event=True,
     )  # consumption is negative, production is positive
+    with pd.option_context(
+            "display.max_rows", None, "display.max_columns", None, "display.width", 2000
+    ):
+        print(bdf)
+        print(bdf.sensor)
     df = simplify_index(bdf)
     df = df.reindex(initialize_index(query_window[0], query_window[1], resolution))
     nan_values = df.isnull().values

From ffec586d9eefa22b98706b39120720c6600c6ccc Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 15:53:00 +0100
Subject: [PATCH 37/44] Remove debug statements

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/utils.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py
index b7a4c6770..d4442583d 100644
--- a/flexmeasures/data/models/planning/utils.py
+++ b/flexmeasures/data/models/planning/utils.py
@@ -182,11 +182,6 @@ def get_prices(
         most_recent_beliefs_only=True,
         one_deterministic_belief_per_event=True,
     )
-    with pd.option_context(
-            "display.max_rows", None, "display.max_columns", None, "display.width", 2000
-    ):
-        print(price_bdf)
-        print(price_bdf.sensor)
     price_df = simplify_index(price_bdf)
     nan_prices = price_df.isnull().values
     if nan_prices.all() or price_df.empty:
@@ -241,11 +236,6 @@ def get_power_values(
         most_recent_beliefs_only=True,
         one_deterministic_belief_per_event=True,
     )  # consumption is negative, production is positive
-    with pd.option_context(
-            "display.max_rows", None, "display.max_columns", None, "display.width", 2000
-    ):
-        print(bdf)
-        print(bdf.sensor)
     df = simplify_index(bdf)
     df = df.reindex(initialize_index(query_window[0], query_window[1], resolution))
     nan_values = df.isnull().values

From 66e84cca61cfe4da29ecc62a57240d1fec7f3a7d Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 15:56:13 +0100
Subject: [PATCH 38/44] black

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/time_series.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py
index 638614853..38ddd60f3 100644
--- a/flexmeasures/data/models/time_series.py
+++ b/flexmeasures/data/models/time_series.py
@@ -652,17 +652,17 @@ def search(
             # Workaround (1st half) for https://github.com/FlexMeasures/flexmeasures/issues/484
             if event_starts_after is not None:
                 _event_starts_after = (
-                        event_starts_after
-                        - sensor.event_resolution
-                        + timedelta(milliseconds=1)
+                    event_starts_after
+                    - sensor.event_resolution
+                    + timedelta(milliseconds=1)
                 )
             else:
                 _event_starts_after = event_starts_after
             if event_ends_before is not None:
                 _event_ends_before = (
-                        event_ends_before
-                        + sensor.event_resolution
-                        - timedelta(milliseconds=1)
+                    event_ends_before
+                    + sensor.event_resolution
+                    - timedelta(milliseconds=1)
                 )
             else:
                 _event_ends_before = event_ends_before

From 5bb7751e07f1a643a741edffd21149c9b41c2a06 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Fri, 11 Nov 2022 15:58:36 +0100
Subject: [PATCH 39/44] Correct mistake while refactoring

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/conftest.py    | 2 +-
 flexmeasures/data/models/planning/tests/test_solver.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py
index ebfe12379..acf8ca4ab 100644
--- a/flexmeasures/data/models/planning/tests/conftest.py
+++ b/flexmeasures/data/models/planning/tests/conftest.py
@@ -20,7 +20,7 @@ def setup_planning_test_data(db, add_market_prices, add_charging_station_assets)
 
 
 @pytest.fixture(scope="module")
-def create_test_tariffs(db, setup_accounts, setup_sources) -> Dict[str, Sensor]:
+def create_test_tariffs(db, setup_accounts, setup_sources) -> dict[str, Sensor]:
     """Create a fixed consumption tariff and a fixed feed-in tariff that is lower."""
 
     market_type = GenericAssetType(
diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 02bcebb4d..1c7c50e20 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -339,6 +339,8 @@ def test_building_solver_day_2(
         end,
         resolution,
         storage_specs=storage_specs,
+        consumption_price_sensor=consumption_price_sensor,
+        production_price_sensor=production_price_sensor,
         inflexible_device_sensors=inflexible_devices.values(),
     )
     soc_schedule = integrate_time_series(schedule, soc_at_start, decimal_precision=6)

From 3dda0e885b23d92c3bfd19fc48af6a0454d6d2c5 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 17 Nov 2022 15:06:54 +0100
Subject: [PATCH 40/44] Check default price sensor in both scenarios

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 1c7c50e20..bdedebc64 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -302,12 +302,13 @@ def test_building_solver_day_2(
     """
     battery = flexible_devices["battery power sensor"]
     building = battery.generic_asset
+    default_consumption_price_sensor = Sensor.query.filter(
+        Sensor.name == "epex_da"
+    ).one_or_none()
+    assert battery.get_attribute("market_id") == default_consumption_price_sensor.id
     if market_scenario == "dynamic contract":
-        consumption_price_sensor = Sensor.query.filter(
-            Sensor.name == "epex_da"
-        ).one_or_none()
+        consumption_price_sensor = default_consumption_price_sensor
         production_price_sensor = consumption_price_sensor
-        assert battery.get_attribute("market_id") == consumption_price_sensor.id
     elif market_scenario == "fixed contract":
         consumption_price_sensor = create_test_tariffs["consumption_price_sensor"]
         production_price_sensor = create_test_tariffs["production_price_sensor"]

From 2fac4f5b09761f1b8acf246f206aaa5d72567b0e Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 17 Nov 2022 15:09:16 +0100
Subject: [PATCH 41/44] Expand explanation of optimization context

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/planning/tests/test_solver.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index bdedebc64..91591fd1f 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -295,6 +295,8 @@ def test_building_solver_day_2(
     In the test data:
     - Hours with net production coincide with low dynamic market prices.
     - Hours with net consumption coincide with high dynamic market prices.
+    So when the prices are low (in scenario 1), we have net production, and when they are high, net consumption.
+    That means we have first net consumption, then net production, and then net consumption again.
     In either scenario, we expect the scheduler to:
     - completely discharge within the first 8 hours (either due to 1) high prices, or 2) net consumption)
     - completely charge within the next 8 hours (either due to 1) low prices, or 2) net production)
@@ -380,7 +382,8 @@ def test_building_solver_day_2(
         assert soc >= max(soc_min, battery.get_attribute("min_soc_in_mwh"))
         assert soc <= battery.get_attribute("max_soc_in_mwh")
 
-    # Check whether the resulting soc schedule follows our expectations for:
+    # Check whether the resulting soc schedule follows our expectations for.
+    # To recap, in scenario 1 and 2, the schedule should mainly be influenced by:
     # 1) 8 expensive, 8 cheap and 8 expensive hours
     # 2) 8 net-consumption, 8 net-production and 8 net-consumption hours
 

From 8343eba086d181d6a64a1b806697c4d401bfede2 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Thu, 17 Nov 2022 15:13:01 +0100
Subject: [PATCH 42/44] Reorder assertions and add more comments

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 .../data/models/planning/tests/test_solver.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py
index 91591fd1f..08e15334c 100644
--- a/flexmeasures/data/models/planning/tests/test_solver.py
+++ b/flexmeasures/data/models/planning/tests/test_solver.py
@@ -291,7 +291,7 @@ def test_building_solver_day_2(
 ):
     """Check battery scheduling results within the context of a building with PV, for day 2, against the following market scenarios:
     1) a dynamic tariff with equal consumption and feed-in tariffs, that is set up with 8 expensive, then 8 cheap, then again 8 expensive hours.
-    2) a fixed consumption tariff and a fixed feed-in tariff that is lower.
+    2) a fixed consumption tariff and a fixed feed-in tariff that is lower, which incentives to maximize self-consumption of PV power into the battery.
     In the test data:
     - Hours with net production coincide with low dynamic market prices.
     - Hours with net consumption coincide with high dynamic market prices.
@@ -387,20 +387,23 @@ def test_building_solver_day_2(
     # 1) 8 expensive, 8 cheap and 8 expensive hours
     # 2) 8 net-consumption, 8 net-production and 8 net-consumption hours
 
-    # 1) The battery sold out at the end of its planning horizon
-    # 2) The battery discharged as far as it could during the last 8 net-consumption hours
-    assert soc_schedule.iloc[-1] == max(
-        soc_min, battery.get_attribute("min_soc_in_mwh")
-    )
-
+    # Result after 8 hours
     # 1) Sell what you begin with
     # 2) The battery discharged as far as it could during the first 8 net-consumption hours
     assert soc_schedule.loc[start + timedelta(hours=8)] == max(
         soc_min, battery.get_attribute("min_soc_in_mwh")
     )
 
-    # 1) Buy what you can to sell later
+    # Result after second 8 hour-interval
+    # 1) Buy what you can to sell later, when prices will be high again
     # 2) The battery charged with PV power as far as it could during the middle 8 net-production hours
     assert soc_schedule.loc[start + timedelta(hours=16)] == min(
         soc_max, battery.get_attribute("max_soc_in_mwh")
     )
+
+    # Result at end of day
+    # 1) The battery sold out at the end of its planning horizon
+    # 2) The battery discharged as far as it could during the last 8 net-consumption hours
+    assert soc_schedule.iloc[-1] == max(
+        soc_min, battery.get_attribute("min_soc_in_mwh")
+    )

From 17148d6bd195c52cf412ed3b72e94fa63bd07ba1 Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Nov 2022 22:19:38 +0100
Subject: [PATCH 43/44] Upgrade timely-beliefs to partly resolve workaround

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 flexmeasures/data/models/time_series.py | 22 +++-------------------
 requirements/app.in                     |  2 +-
 requirements/app.txt                    |  2 +-
 3 files changed, 5 insertions(+), 21 deletions(-)

diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py
index 38ddd60f3..d1acf04f6 100644
--- a/flexmeasures/data/models/time_series.py
+++ b/flexmeasures/data/models/time_series.py
@@ -649,28 +649,12 @@ def search(
 
         bdf_dict = {}
         for sensor in sensors:
-            # Workaround (1st half) for https://github.com/FlexMeasures/flexmeasures/issues/484
-            if event_starts_after is not None:
-                _event_starts_after = (
-                    event_starts_after
-                    - sensor.event_resolution
-                    + timedelta(milliseconds=1)
-                )
-            else:
-                _event_starts_after = event_starts_after
-            if event_ends_before is not None:
-                _event_ends_before = (
-                    event_ends_before
-                    + sensor.event_resolution
-                    - timedelta(milliseconds=1)
-                )
-            else:
-                _event_ends_before = event_ends_before
             bdf = cls.search_session(
                 session=db.session,
                 sensor=sensor,
-                event_starts_after=_event_starts_after,
-                event_ends_before=_event_ends_before,
+                # Workaround (1st half) for https://github.com/FlexMeasures/flexmeasures/issues/484
+                event_ends_after=event_starts_after,
+                event_starts_before=event_ends_before,
                 beliefs_after=beliefs_after,
                 beliefs_before=beliefs_before,
                 horizons_at_least=horizons_at_least,
diff --git a/requirements/app.in b/requirements/app.in
index 82e15f9ec..df6a9a220 100644
--- a/requirements/app.in
+++ b/requirements/app.in
@@ -28,7 +28,7 @@ tldextract
 pyomo>=5.6
 tabulate
 timetomodel>=0.7.1
-timely-beliefs[forecast]>=1.13
+timely-beliefs[forecast]>=1.14
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata
diff --git a/requirements/app.txt b/requirements/app.txt
index 71df651f4..480874aa5 100644
--- a/requirements/app.txt
+++ b/requirements/app.txt
@@ -314,7 +314,7 @@ tabulate==0.8.10
     # via -r requirements/app.in
 threadpoolctl==3.1.0
     # via scikit-learn
-timely-beliefs[forecast]==1.13.0
+timely-beliefs[forecast]==1.14.0
     # via -r requirements/app.in
 timetomodel==0.7.1
     # via -r requirements/app.in

From f84c200e7039c9c3d55e5cd1303943522124b06c Mon Sep 17 00:00:00 2001
From: "F.N. Claessen" <felix@seita.nl>
Date: Mon, 21 Nov 2022 22:29:41 +0100
Subject: [PATCH 44/44] changelog entry

Signed-off-by: F.N. Claessen <felix@seita.nl>
---
 documentation/changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/documentation/changelog.rst b/documentation/changelog.rst
index 3261f28a0..8c8dcbaa8 100644
--- a/documentation/changelog.rst
+++ b/documentation/changelog.rst
@@ -32,6 +32,7 @@ Infrastructure / Support
 * Plugins can save BeliefsSeries, too, instead of just BeliefsDataFrames [see `PR #523 <http://www.github.com/FlexMeasures/flexmeasures/pull/523>`_]
 * Improve documentation and code w.r.t. storage flexibility modelling ― prepare for handling other schedulers & merge battery and car charging schedulers [see `PR #511 <http://www.github.com/FlexMeasures/flexmeasures/pull/511>`_]
 * Revised strategy for removing unchanged beliefs when saving data: retain the oldest measurement (ex-post belief), too [see `PR #518 <http://www.github.com/FlexMeasures/flexmeasures/pull/518>`_]
+* Scheduling test for maximizing self-consumption, and improved time series db queries for fixed tariffs (and other long-term constants) [see `PR #532 <http://www.github.com/FlexMeasures/flexmeasures/pull/532>`_]
 
 
 v0.11.3 | November 2, 2022