From f64709a78dd84341b621cb9898d497b6b59d1865 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:24:53 +0200 Subject: [PATCH 01/48] test: add shiftable_load fixture Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/tests/conftest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index acf8ca4ab..daf66eb00 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -187,6 +187,22 @@ def add_inflexible_device_forecasts( } +@pytest.fixture(scope="module") +def shiftable_load(db, building, setup_sources) -> dict[str, Sensor]: + """ + Set up a shiftable load sensor where the output of the optimization will live + """ + _shiftable_load = Sensor( + name="Shiftable Load", + generic_asset=building, + event_resolution=timedelta(hours=1), + unit="kW", + ) + db.session.add(_shiftable_load) + + return _shiftable_load + + def add_as_beliefs(db, sensor, values, time_slots, source): beliefs = [ TimedBelief( From 35ca7e7af0b95496903943b7b8a96c1e887bbc32 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:26:19 +0200 Subject: [PATCH 02/48] test: move fixture setup_dummy_sensors from test_reporting.py to conftest.py Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/tests/conftest.py | 30 +++++++++++++++++++ .../data/schemas/tests/test_reporting.py | 29 ------------------ 2 files changed, 30 insertions(+), 29 deletions(-) create mode 100644 flexmeasures/data/schemas/tests/conftest.py diff --git a/flexmeasures/data/schemas/tests/conftest.py b/flexmeasures/data/schemas/tests/conftest.py new file mode 100644 index 000000000..58f947ec3 --- /dev/null +++ b/flexmeasures/data/schemas/tests/conftest.py @@ -0,0 +1,30 @@ +import pytest +from datetime import timedelta + +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType + + +@pytest.fixture(scope="module") +def setup_dummy_sensors(db, app): + + dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") + db.session.add(dummy_asset_type) + + dummy_asset = GenericAsset( + name="DummyGenericAsset", generic_asset_type=dummy_asset_type + ) + db.session.add(dummy_asset) + + sensor1 = Sensor( + "sensor 1", generic_asset=dummy_asset, event_resolution=timedelta(hours=1) + ) + db.session.add(sensor1) + sensor2 = Sensor( + "sensor 2", generic_asset=dummy_asset, event_resolution=timedelta(hours=1) + ) + db.session.add(sensor2) + + db.session.commit() + + yield sensor1, sensor2 diff --git a/flexmeasures/data/schemas/tests/test_reporting.py b/flexmeasures/data/schemas/tests/test_reporting.py index cb5490052..8d30e0013 100644 --- a/flexmeasures/data/schemas/tests/test_reporting.py +++ b/flexmeasures/data/schemas/tests/test_reporting.py @@ -1,6 +1,3 @@ -from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType - from flexmeasures.data.schemas.reporting.pandas_reporter import ( PandasReporterConfigSchema, ) @@ -9,32 +6,6 @@ import pytest -@pytest.fixture(scope="module") -def setup_dummy_sensors(db, app): - - dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") - db.session.add(dummy_asset_type) - - dummy_asset = GenericAsset( - name="DummyGenericAsset", generic_asset_type=dummy_asset_type - ) - db.session.add(dummy_asset) - - sensor1 = Sensor("sensor 1", generic_asset=dummy_asset) - db.session.add(sensor1) - sensor2 = Sensor("sensor 2", generic_asset=dummy_asset) - db.session.add(sensor2) - - db.session.commit() - - yield sensor1, sensor2 - - db.session.delete(sensor1) - db.session.delete(sensor2) - - db.session.commit() - - @pytest.mark.parametrize( "reporter_config, is_valid", [ From 99af08c95b580d7844638e08b108e5294bcc2c8f Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:27:25 +0200 Subject: [PATCH 03/48] feat: add ShiftableLoadFlexModelSchema Signed-off-by: Victor Garcia Reolid --- .../data/schemas/scheduling/shiftable_load.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 flexmeasures/data/schemas/scheduling/shiftable_load.py diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/shiftable_load.py new file mode 100644 index 000000000..aa09a98ea --- /dev/null +++ b/flexmeasures/data/schemas/scheduling/shiftable_load.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from datetime import datetime +import pytz +import pandas as pd + +from marshmallow import ( + Schema, + post_load, + fields, + pre_load, +) + +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField + + +from flexmeasures.data.schemas.sensors import SensorIdField + +from enum import Enum + + +class LoadType(Enum): + INFLEXIBLE = "INFLEXIBLE" + BREAKABLE = "BREAKABLE" + SHIFTABLE = "SHIFTABLE" + + +class OptimizationSense(Enum): + MAX = "MAX" + MIN = "MIN" + + +class TimeIntervalSchema(Schema): + start = AwareDateTimeField(required=True) + duration = DurationField(required=True) + + +class ShiftableLoadFlexModelSchema(Schema): + cost_sensor = SensorIdField(data_key="cost-sensor") + duration = DurationField(required=True) + power = fields.Float(required=True) + + load_type = fields.Enum( + LoadType, load_default=LoadType.INFLEXIBLE, data_key="load-type" + ) + time_restrictions = fields.List( + fields.Nested(TimeIntervalSchema()), + data_key="time-restrictions", + load_default=[], + ) + optimization_sense = fields.Enum( + OptimizationSense, + load_default=OptimizationSense.MIN, + data_key="optimization-sense", + ) + + def __init__(self, sensor: Sensor, start: datetime, end: datetime, *args, **kwargs): + """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" + self.start = start.astimezone(pytz.utc) + self.end = end.astimezone(pytz.utc) + self.sensor = sensor + super().__init__(*args, **kwargs) + + def get_mask_from_events(self, events: list[dict[str, str]] | None) -> pd.Series: + """Convert events to a mask of the time periods that are valid + + :param events: list of events defined as dictionaries with a start and duration + :return: mask of the allowed time periods + """ + series = pd.Series( + index=pd.date_range( + self.start, + self.end, + freq=self.sensor.event_resolution, + inclusive="left", + name="event_start", + tz=self.start.tzinfo, + ), + data=False, + ) + + if events is None: + return series + + for event in events: + start = event["start"] + duration = event["duration"] + end = start + duration + series[(series.index >= start) & (series.index < end)] = True + + return series + + @post_load + def post_load_time_restrictions(self, data: dict, **kwargs) -> dict: + """Convert events (list of [start, duration] pairs) into a mask (pandas Series)""" + + data["time_restrictions"] = self.get_mask_from_events(data["time_restrictions"]) + + return data + + @pre_load + def pre_load_load_type(self, data: dict, **kwargs) -> dict: + """Fallback mechanism for the load_type variable. If not found in data, + it tries to find it in among the sensor attributes and, if it's not found + there either, it defaults to "INFLEXIBLE". + """ + if "load-type" not in data or data["load-type"] is None: + load_type = self.sensor.get_attribute("load_type") + + if load_type is None: + load_type = "INFLEXIBLE" + + data["load-type"] = load_type + + return data From 5216fa32e99903f1ccfde3605cdb838e68ae6720 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:28:05 +0200 Subject: [PATCH 04/48] test: add ShiftableLoadFlexModelSchema tests Signed-off-by: Victor Garcia Reolid --- .../data/schemas/tests/test_scheduling.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 flexmeasures/data/schemas/tests/test_scheduling.py diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py new file mode 100644 index 000000000..e0d054e7c --- /dev/null +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -0,0 +1,75 @@ +from flexmeasures.data.schemas.scheduling.shiftable_load import ( + ShiftableLoadFlexModelSchema, + LoadType, +) + +from datetime import datetime +import pytz + + +def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): + + sensor1, _ = setup_dummy_sensors + + schema = ShiftableLoadFlexModelSchema( + sensor=sensor1, + start=datetime(2023, 1, 1, tzinfo=pytz.UTC), + end=datetime(2023, 1, 2, tzinfo=pytz.UTC), + ) + + shiftable_load_flex_model = schema.load( + { + "cost-sensor": sensor1.id, + "duration": "PT4H", + "time-restrictions": [ + {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} + ], + } + ) + + print(shiftable_load_flex_model) + + +def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): + + sensor1, _ = setup_dummy_sensors + + # checking default + + schema = ShiftableLoadFlexModelSchema( + sensor=sensor1, + start=datetime(2023, 1, 1, tzinfo=pytz.UTC), + end=datetime(2023, 1, 2, tzinfo=pytz.UTC), + ) + + shiftable_load_flex_model = schema.load( + { + "cost-sensor": sensor1.id, + "duration": "PT4H", + "time-restrictions": [ + {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} + ], + } + ) + + assert shiftable_load_flex_model["load_type"] == LoadType.INFLEXIBLE + + sensor1.attributes["load_type"] = "SHIFTABLE" + + schema = ShiftableLoadFlexModelSchema( + sensor=sensor1, + start=datetime(2023, 1, 1, tzinfo=pytz.UTC), + end=datetime(2023, 1, 2, tzinfo=pytz.UTC), + ) + + shiftable_load_flex_model = schema.load( + { + "duration": "PT4H", + "cost-sensor": sensor1.id, + "time-restrictions": [ + {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} + ], + } + ) + + assert shiftable_load_flex_model["load_type"] == LoadType.SHIFTABLE From 8faf7e56019f809fc7febcd2096e7744f3c1a6f0 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:29:01 +0200 Subject: [PATCH 05/48] feat: add ShiftableLoadScheduler Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 flexmeasures/data/models/planning/shiftable_load.py diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py new file mode 100644 index 000000000..ee63e725c --- /dev/null +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from math import ceil +from datetime import timedelta +import pytz + +import pandas as pd + +from flask import current_app + +from flexmeasures.data.models.planning import Scheduler + +from flexmeasures.data.queries.utils import simplify_index +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.schemas.scheduling.shiftable_load import ( + ShiftableLoadFlexModelSchema, + LoadType, + OptimizationSense, +) +from flexmeasures.data.schemas.scheduling import FlexContextSchema + + +class ShiftableLoadScheduler(Scheduler): + + __version__ = "1" + __author__ = "Seita" + + def compute(self) -> pd.Series | None: + if not self.config_deserialized: + self.deserialize_config() + + start = self.start.astimezone(pytz.utc) + end = self.end.astimezone(pytz.utc) + resolution = self.resolution + belief_time = self.belief_time + sensor = self.sensor + + cost_sensor: Sensor = self.flex_model.get("cost_sensor") + duration: timedelta = self.flex_model.get("duration") + power = self.flex_model.get("power") + optimization_sense = self.flex_model.get("optimization_sense") + load_type: LoadType = self.flex_model.get("load_type") + time_restrictions = self.flex_model.get("time_restrictions") + + cost = cost_sensor.search_beliefs( + event_starts_after=start, + event_ends_before=end, + resolution=resolution, + one_deterministic_belief_per_event=True, + beliefs_before=belief_time, + ) + cost = simplify_index(cost) + + # Create an empty schedule + schedule = pd.Series( + index=pd.date_range( + start, + end, + freq=sensor.event_resolution, + closed="left", + name="event_start", + ), + data=0, + name="event_value", + ) + + # Optimize schedule for tomorrow. We can fill len(schedule) rows, at most. + rows_to_fill = min(ceil(duration / cost_sensor.event_resolution), len(schedule)) + + if rows_to_fill > len(schedule): + current_app.logger.warning( + f"Duration of the period exceeds the schedule window. The resulting schedule will be trimmed to fit the planning window ({start}, {end})." + ) + + assert rows_to_fill >= 1, "" + + if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: + # get start time instants that are not feasible, i.e, some time during the ON period goes through + # a time restriction interval. + time_restrictions = ( + time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) + ) + time_restrictions = (time_restrictions == 1) | time_restrictions.isna() + + if time_restrictions.sum() == len(time_restrictions): + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." + ) + else: # LoadType.BREAKABLE + if (~time_restrictions).sum() < rows_to_fill: + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." + ) + + if load_type == LoadType.INFLEXIBLE: + start = time_restrictions[~time_restrictions].index[0] + + # Schedule as early as possible + schedule.loc[ + start : start + sensor.event_resolution * (rows_to_fill - 1) + ] = power + + elif load_type == LoadType.BREAKABLE: + cost = cost[~time_restrictions].reset_index() + + if optimization_sense == OptimizationSense.MIN: + cost_ranking = cost.sort_values( + by=["event_value", "event_start"], ascending=[True, True] + ) + else: + cost_ranking = cost.sort_values( + by=["event_value", "event_start"], ascending=[False, True] + ) + + # Break up schedule and divide it over the cleanest time slots + schedule.loc[cost_ranking.head(rows_to_fill).event_start] = power + + elif load_type == LoadType.SHIFTABLE: + block_cost = simplify_index( + cost.rolling(rows_to_fill).sum().shift(-rows_to_fill + 1) + ) + + if optimization_sense == OptimizationSense.MIN: + start = block_cost[~time_restrictions].idxmin() + else: + start = block_cost[~time_restrictions].idxmax() + + start = start.event_value + + schedule.loc[ + start : start + sensor.event_resolution * (rows_to_fill - 1) + ] = power + + else: + raise ValueError(f"Unknown load type '{load_type}'") + + return schedule.tz_convert(self.start.tzinfo) + + def deserialize_flex_config(self): + """ """ + if self.flex_model is None: + self.flex_model = {} + + self.flex_model = ShiftableLoadFlexModelSchema( + start=self.start, end=self.end, sensor=self.sensor + ).load(self.flex_model) + + self.flex_context = FlexContextSchema().load(self.flex_context) From b1472a5c9b0a2619cbc444eb92cb00b21a83a41c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:30:13 +0200 Subject: [PATCH 06/48] tests: add ShiftableLoadScheduler tests Signed-off-by: Victor Garcia Reolid --- .../planning/tests/test_shiftable_load.py | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 flexmeasures/data/models/planning/tests/test_shiftable_load.py diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_shiftable_load.py new file mode 100644 index 000000000..e1ca8e64b --- /dev/null +++ b/flexmeasures/data/models/planning/tests/test_shiftable_load.py @@ -0,0 +1,205 @@ +from datetime import datetime, timedelta +import pytest +import pytz + +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.models.planning.shiftable_load import ShiftableLoadScheduler + + +tz = pytz.timezone("Europe/Amsterdam") +start = tz.localize(datetime(2015, 1, 2)) +end = tz.localize(datetime(2015, 1, 3)) +resolution = timedelta(hours=1) + + +@pytest.mark.parametrize( + "load_type, optimal_start", + [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], +) +def test_shiftable_scheduler( + add_battery_assets, shiftable_load, load_type, optimal_start +): + """ """ + + # get the sensors from the database + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + flex_model = { + "cost-sensor": epex_da.id, + "duration": "PT4H", + "load-type": load_type, + "power": 4, + } + + scheduler = ShiftableLoadScheduler( + shiftable_load, + start, + end, + resolution, + flex_model=flex_model, + ) + schedule = scheduler.compute() + + optimal_start = tz.localize(optimal_start) + + mask = (optimal_start <= schedule.index) & ( + schedule.index < optimal_start + timedelta(hours=4) + ) + + assert (schedule[mask] == 4).all() + assert (schedule[~mask] == 0).all() + + +@pytest.mark.parametrize( + "load_type, optimal_start", + [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], +) +def test_duration_exceeds_planning_window( + add_battery_assets, shiftable_load, load_type, optimal_start +): + """ """ + + # get the sensors from the database + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + flex_model = { + "cost-sensor": epex_da.id, + "duration": "PT48H", + "load-type": load_type, + "power": 4, + } + + scheduler = ShiftableLoadScheduler( + shiftable_load, + start, + end, + resolution, + flex_model=flex_model, + ) + schedule = scheduler.compute() + + optimal_start = tz.localize(optimal_start) + + assert (schedule == 4).all() + + +def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_load): + """ """ + + # get the sensors from the database + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + # time parameters + + flex_model = { + "cost-sensor": epex_da.id, + "duration": "PT4H", + "load-type": "SHIFTABLE", + "power": 4, + "time-restrictions": [ + {"start": "2015-01-02T08:00:00+01:00", "duration": "PT2H"} + ], + } + + scheduler = ShiftableLoadScheduler( + shiftable_load, + start, + end, + resolution, + flex_model=flex_model, + ) + schedule = scheduler.compute() + + optimal_start = tz.localize(datetime(2015, 1, 2, 10)) + + mask = (optimal_start <= schedule.index) & ( + schedule.index < optimal_start + timedelta(hours=4) + ) + + assert (schedule[mask] == 4).all() + assert (schedule[~mask] == 0).all() + + # check that the time restrictions are fulfilled + time_restrictions = scheduler.flex_model["time_restrictions"] + time_restrictions = time_restrictions.tz_convert(tz) + + assert (schedule[time_restrictions] == 0).all() + + +def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_load): + """ """ + + # get the sensors from the database + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + # time parameters + + flex_model = { + "cost-sensor": epex_da.id, + "duration": "PT4H", + "load-type": "BREAKABLE", + "power": 4, + "time-restrictions": [ + {"start": "2015-01-02T09:00:00+01:00", "duration": "PT1H"}, + {"start": "2015-01-02T11:00:00+01:00", "duration": "PT1H"}, + {"start": "2015-01-02T13:00:00+01:00", "duration": "PT1H"}, + {"start": "2015-01-02T15:00:00+01:00", "duration": "PT1H"}, + ], + } + + scheduler = ShiftableLoadScheduler( + shiftable_load, + start, + end, + resolution, + flex_model=flex_model, + ) + schedule = scheduler.compute() + + expected_schedule = [0] * 8 + [4, 0, 4, 0, 4, 0, 4, 0] + [0] * 8 + + assert (schedule == expected_schedule).all() + + # check that the time restrictions are fulfilled + time_restrictions = scheduler.flex_model["time_restrictions"] + time_restrictions = time_restrictions.tz_convert(tz) + + assert (schedule[time_restrictions] == 0).all() + + +@pytest.mark.parametrize( + "load_type, time_restrictions", + [ + ("BREAKABLE", [{"start": "2015-01-02T00:00:00+01:00", "duration": "PT24H"}]), + ("INFLEXIBLE", [{"start": "2015-01-02T03:00:00+01:00", "duration": "PT21H"}]), + ("SHIFTABLE", [{"start": "2015-01-02T03:00:00+01:00", "duration": "PT21H"}]), + ], +) +def test_impossible_schedules( + add_battery_assets, shiftable_load, load_type, time_restrictions +): + """ """ + + # get the sensors from the database + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + # time parameters + + flex_model = { + "cost-sensor": epex_da.id, + "duration": "PT4H", + "load-type": load_type, + "power": 4, + "time-restrictions": time_restrictions, + } + + scheduler = ShiftableLoadScheduler( + shiftable_load, + start, + end, + resolution, + flex_model=flex_model, + ) + + with pytest.raises(ValueError): + scheduler.compute() From e32767a7fe5a8526aa94a94d81f275906aaccbc7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 14 Jun 2023 16:33:56 +0200 Subject: [PATCH 07/48] test: add required parameter Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/tests/test_scheduling.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index e0d054e7c..52d94ac63 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -21,6 +21,7 @@ def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): { "cost-sensor": sensor1.id, "duration": "PT4H", + "power": 30.0, "time-restrictions": [ {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} ], @@ -46,6 +47,7 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): { "cost-sensor": sensor1.id, "duration": "PT4H", + "power": 30.0, "time-restrictions": [ {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} ], @@ -65,6 +67,7 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): shiftable_load_flex_model = schema.load( { "duration": "PT4H", + "power": 30.0, "cost-sensor": sensor1.id, "time-restrictions": [ {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} From d26cd53bd9fc4bb4fa3a56f193f8081bae650cc6 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 15 Jun 2023 11:32:08 +0200 Subject: [PATCH 08/48] docs: improve docstrings Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 155 +++++++++++++----- .../data/models/planning/tests/conftest.py | 4 +- .../planning/tests/test_shiftable_load.py | 27 ++- .../data/schemas/scheduling/shiftable_load.py | 4 +- 4 files changed, 137 insertions(+), 53 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index ee63e725c..b8a67374f 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -6,8 +6,6 @@ import pandas as pd -from flask import current_app - from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.queries.utils import simplify_index @@ -26,6 +24,31 @@ class ShiftableLoadScheduler(Scheduler): __author__ = "Seita" def compute(self) -> pd.Series | None: + """Schedule a load, defined as a `power` and a `duration`, within the specified time window. + For example, this scheduler can plan the start of a process that lasts 5h and requires a power of 10kW. + + This scheduler supports three types of `load_types`: + - Inflexible: this load requires to be scheduled as soon as possible. + - Breakable: this load can be divisible in smaller consumption periods. + - Shiftable: this load can start at any time within the specified time window. + + The resulting schedule provides the power flow at each time period. + + Parameters + ========== + + cost_sensor: it defines the utility (economic, environmental, ) in each + time period. It has units of quantity/energy, for example, EUR/kWh. + power: nominal power of the load. + duration: time that the load lasts. + + optimization_sense: objective of the scheduler, to maximize or minimize. + time_restrictions: time periods in which the load cannot be schedule to. + load_type: Inflexible, Breakable or Shiftable. + + :returns: The computed schedule. + """ + if not self.config_deserialized: self.deserialize_config() @@ -42,6 +65,7 @@ def compute(self) -> pd.Series | None: load_type: LoadType = self.flex_model.get("load_type") time_restrictions = self.flex_model.get("time_restrictions") + # get cost data cost = cost_sensor.search_beliefs( event_starts_after=start, event_ends_before=end, @@ -51,7 +75,7 @@ def compute(self) -> pd.Series | None: ) cost = simplify_index(cost) - # Create an empty schedule + # create an empty schedule schedule = pd.Series( index=pd.date_range( start, @@ -64,19 +88,22 @@ def compute(self) -> pd.Series | None: name="event_value", ) - # Optimize schedule for tomorrow. We can fill len(schedule) rows, at most. + # optimize schedule for tomorrow. We can fill len(schedule) rows, at most rows_to_fill = min(ceil(duration / cost_sensor.event_resolution), len(schedule)) + # convert power to energy using the resolution of the sensor. + # e.g. resolution=15min, power=1kW -> energy=250W + energy = power * cost_sensor.event_resolution / timedelta(hours=1) + if rows_to_fill > len(schedule): - current_app.logger.warning( + raise ValueError( f"Duration of the period exceeds the schedule window. The resulting schedule will be trimmed to fit the planning window ({start}, {end})." ) - assert rows_to_fill >= 1, "" - + # check if the time_restrictions allow for a load of the duration provided if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: - # get start time instants that are not feasible, i.e, some time during the ON period goes through - # a time restriction interval. + # get start time instants that are not feasible, i.e. some time during the ON period goes through + # a time restriction interval time_restrictions = ( time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) ) @@ -92,52 +119,94 @@ def compute(self) -> pd.Series | None: "Cannot allocate a block of time {duration} given the time restrictions provided." ) + # create schedule if load_type == LoadType.INFLEXIBLE: - start = time_restrictions[~time_restrictions].index[0] - - # Schedule as early as possible - schedule.loc[ - start : start + sensor.event_resolution * (rows_to_fill - 1) - ] = power - + self.compute_inflexible(schedule, time_restrictions, rows_to_fill, energy) elif load_type == LoadType.BREAKABLE: - cost = cost[~time_restrictions].reset_index() - - if optimization_sense == OptimizationSense.MIN: - cost_ranking = cost.sort_values( - by=["event_value", "event_start"], ascending=[True, True] - ) - else: - cost_ranking = cost.sort_values( - by=["event_value", "event_start"], ascending=[False, True] - ) - - # Break up schedule and divide it over the cleanest time slots - schedule.loc[cost_ranking.head(rows_to_fill).event_start] = power - + self.compute_breakable( + schedule, + optimization_sense, + time_restrictions, + cost, + rows_to_fill, + energy, + ) elif load_type == LoadType.SHIFTABLE: - block_cost = simplify_index( - cost.rolling(rows_to_fill).sum().shift(-rows_to_fill + 1) + self.compute_shiftable( + schedule, + optimization_sense, + time_restrictions, + cost, + rows_to_fill, + energy, ) + else: + raise ValueError(f"Unknown load type '{load_type}'") - if optimization_sense == OptimizationSense.MIN: - start = block_cost[~time_restrictions].idxmin() - else: - start = block_cost[~time_restrictions].idxmax() + return schedule.tz_convert(self.start.tzinfo) - start = start.event_value + def compute_inflexible( + self, + schedule: pd.Series, + time_restrictions: pd.Series, + rows_to_fill: int, + energy: float, + ) -> None: + """Schedule load as early as possible.""" + start = time_restrictions[~time_restrictions].index[0] + + schedule.loc[start : start + self.resolution * (rows_to_fill - 1)] = energy + + def compute_breakable( + self, + schedule: pd.Series, + optimization_sense: OptimizationSense, + time_restrictions: pd.Series, + cost: pd.DataFrame, + rows_to_fill: int, + energy: float, + ) -> None: + """Break up schedule and divide it over the time slots with the largest utility (max/min cost depending on optimization_sense).""" + cost = cost[~time_restrictions].reset_index() + + if optimization_sense == OptimizationSense.MIN: + cost_ranking = cost.sort_values( + by=["event_value", "event_start"], ascending=[True, True] + ) + else: + cost_ranking = cost.sort_values( + by=["event_value", "event_start"], ascending=[False, True] + ) - schedule.loc[ - start : start + sensor.event_resolution * (rows_to_fill - 1) - ] = power + schedule.loc[cost_ranking.head(rows_to_fill).event_start] = energy + + def compute_shiftable( + self, + schedule: pd.Series, + optimization_sense: OptimizationSense, + time_restrictions: pd.Series, + cost: pd.DataFrame, + rows_to_fill: int, + energy: float, + ) -> None: + """Schedules a block of consumption/production of `rows_to_fill` periods to maximize a utility.""" + block_cost = simplify_index( + cost.rolling(rows_to_fill).sum().shift(-rows_to_fill + 1) + ) + if optimization_sense == OptimizationSense.MIN: + start = block_cost[~time_restrictions].idxmin() else: - raise ValueError(f"Unknown load type '{load_type}'") + start = block_cost[~time_restrictions].idxmax() - return schedule.tz_convert(self.start.tzinfo) + start = start.event_value + + schedule.loc[start : start + self.resolution * (rows_to_fill - 1)] = energy def deserialize_flex_config(self): - """ """ + """Deserialize flex_model using the schema ShiftableLoadFlexModelSchema and + flex_context using FlexContextSchema + """ if self.flex_model is None: self.flex_model = {} diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index daf66eb00..85d33c229 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -190,13 +190,13 @@ def add_inflexible_device_forecasts( @pytest.fixture(scope="module") def shiftable_load(db, building, setup_sources) -> dict[str, Sensor]: """ - Set up a shiftable load sensor where the output of the optimization will live + Set up a shiftable load sensor where the output of the optimization is stored. """ _shiftable_load = Sensor( name="Shiftable Load", generic_asset=building, event_resolution=timedelta(hours=1), - unit="kW", + unit="kWh", ) db.session.add(_shiftable_load) diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_shiftable_load.py index e1ca8e64b..704cc52e7 100644 --- a/flexmeasures/data/models/planning/tests/test_shiftable_load.py +++ b/flexmeasures/data/models/planning/tests/test_shiftable_load.py @@ -19,7 +19,10 @@ def test_shiftable_scheduler( add_battery_assets, shiftable_load, load_type, optimal_start ): - """ """ + """ + Test scheduling a load of 4kW of power that last 4h using the ShiftableLoadScheduler + without time restrictions. + """ # get the sensors from the database epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() @@ -57,7 +60,9 @@ def test_shiftable_scheduler( def test_duration_exceeds_planning_window( add_battery_assets, shiftable_load, load_type, optimal_start ): - """ """ + """ + Test scheduling a load that last longer than the planning window. + """ # get the sensors from the database epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() @@ -84,7 +89,10 @@ def test_duration_exceeds_planning_window( def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_load): - """ """ + """ + Test ShiftableLoadScheduler with a time restrictions consisting of a block of 2h starting + at 8am. The resulting schedules avoid the 8am-10am period and schedules for a valid period. + """ # get the sensors from the database epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() @@ -127,7 +135,11 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_load): - """ """ + """ + Test breakable load_type of ShiftableLoadScheduler by introducing four 1-hour restrictions + interspaced by 1 hour. The equivalent mask would be the following: [0,...,0,1,0,1,0,1,0,1,0, ...,0]. + This makes the schedule choose time periods between. + """ # get the sensors from the database epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() @@ -178,13 +190,14 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa def test_impossible_schedules( add_battery_assets, shiftable_load, load_type, time_restrictions ): - """ """ + """ + Test schedules with time restrictions that make a 4h block not fit anytime during the + planned window. + """ # get the sensors from the database epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() - # time parameters - flex_model = { "cost-sensor": epex_da.id, "duration": "PT4H", diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/shiftable_load.py index aa09a98ea..d189a2583 100644 --- a/flexmeasures/data/schemas/scheduling/shiftable_load.py +++ b/flexmeasures/data/schemas/scheduling/shiftable_load.py @@ -56,7 +56,9 @@ class ShiftableLoadFlexModelSchema(Schema): ) def __init__(self, sensor: Sensor, start: datetime, end: datetime, *args, **kwargs): - """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" + """Pass start and end to convert time_restrictions into a time series and sensor + as a fallback mechanism for the load_type + """ self.start = start.astimezone(pytz.utc) self.end = end.astimezone(pytz.utc) self.sensor = sensor From d9151df85bd61a6b11339b3acd812a099c7bcd65 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 20 Jul 2023 15:49:45 +0200 Subject: [PATCH 09/48] fix: pandas 2.0 deprecated argument Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/shiftable_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index b8a67374f..78b171cb3 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -81,7 +81,7 @@ def compute(self) -> pd.Series | None: start, end, freq=sensor.event_resolution, - closed="left", + inclusive="left", name="event_start", ), data=0, From 48eed716c401c24956bdc5d78df44c07b880767b Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 20 Jul 2023 15:49:45 +0200 Subject: [PATCH 10/48] fix: pandas 2.0 deprecated argument Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/shiftable_load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index b8a67374f..78b171cb3 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -81,7 +81,7 @@ def compute(self) -> pd.Series | None: start, end, freq=sensor.event_resolution, - closed="left", + inclusive="left", name="event_start", ), data=0, From b54af10c7ae8fa2511e948dd4bd15ce3b38c1057 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 20 Jul 2023 16:10:01 +0200 Subject: [PATCH 11/48] feat: add minimum valuable version of the command flexmeasures add report shiftable. Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 112 ++++++++++++++++++++++++ flexmeasures/cli/tests/conftest.py | 38 ++++++++ flexmeasures/cli/tests/test_data_add.py | 44 ++++++++++ 3 files changed, 194 insertions(+) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index cf31026bf..8b4515f59 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1158,6 +1158,118 @@ def add_schedule_for_storage( click.secho("New schedule is stored.", **MsgStyle.SUCCESS) +@create_schedule.command("shiftable") +@with_appcontext +@click.option( + "--sensor-id", + "power_sensor", + type=SensorIdField(), + required=True, + help="Create schedule for this sensor. Should be a power sensor. Follow up with the sensor's ID.", +) +@click.option( + "--consumption-price-sensor", + "consumption_price_sensor", + type=SensorIdField(), + required=False, + help="Optimize consumption against this sensor. The sensor typically records an electricity price (e.g. in EUR/kWh), but this field can also be used to optimize against some emission intensity factor (e.g. in kg CO₂ eq./kWh). Follow up with the sensor's ID.", +) +@click.option( + "--start", + "start", + type=AwareDateTimeField(format="iso"), + required=True, + help="Schedule starts at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", +) +@click.option( + "--duration", + "duration", + type=DurationField(), + required=True, + help="Duration of schedule, after --start. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", +) +@click.option( + "--load-duration", + "load_duration", + type=DurationField(), + required=True, + help="Duration of the load. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", +) +@click.option( + "--load-type", + "load_type", + type=click.Choice(["INFLEXIBLE", "BREAKABLE", "SHIFTABLE"], case_sensitive=False), + required=False, + default="INFLEXIBLE", + help="Load shift policy type: INFLEXIBLE, BREAKABLE or SHIFTABLE.", +) +@click.option( + "--load-power", + "load_power", + type=ur.Quantity, + required=True, + help="Constant power of the load during the activation period.", +) +@click.option( + "--as-job", + is_flag=True, + help="Whether to queue a scheduling job instead of computing directly. " + "To process the job, run a worker (on any computer, but configured to the same databases) to process the 'scheduling' queue. Defaults to False.", +) +def add_schedule_shiftable_load( + power_sensor: Sensor, + consumption_price_sensor: Sensor, + start: datetime, + duration: timedelta, + load_duration: timedelta, + load_type: str, + load_power: ur.Quantity, + as_job: bool = False, +): + """Create a new schedule for a shiftable asset. + + Current limitations: + - Only supports consumption blocks. + - Not taking into account grid constraints or other loads. + """ + # Parse input and required sensor attributes + if not power_sensor.measures_power: + click.secho( + f"Sensor with ID {power_sensor.id} is not a power sensor.", + **MsgStyle.ERROR, + ) + raise click.Abort() + + end = start + duration + + load_power = convert_units(load_power.magnitude, load_power.units, "MW") # type: ignore + + scheduling_kwargs = dict( + start=start, + end=end, + belief_time=server_now(), + resolution=power_sensor.event_resolution, + flex_model={ + "cost-sensor": consumption_price_sensor.id, + "duration": pd.Timedelta(load_duration).isoformat(), + "load-type": load_type, + "power": load_power, + }, + ) + + if as_job: + job = create_scheduling_job(sensor=power_sensor, **scheduling_kwargs) + if job: + click.secho( + f"New scheduling job {job.id} has been added to the queue.", + **MsgStyle.SUCCESS, + ) + else: + success = make_schedule(sensor_id=power_sensor.id, **scheduling_kwargs) + if success: + click.secho("New schedule is stored.", **MsgStyle.SUCCESS) + + @fm_add_data.command("report") @with_appcontext @click.option( diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 028012edc..029e34915 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -96,3 +96,41 @@ def reporter_config_raw(app, db, setup_dummy_data): ) return reporter_config_raw + + +@pytest.mark.skip_github +@pytest.fixture(scope="module") +def shiftable_load_power_sensor(db, app): + """ + Create an asset with two sensors (1 and 2), and add the same set of 200 beliefs with an hourly resolution to each of them. + Return the two sensors and a result sensor (which has no data). + """ + + shiftable_load_asset_type = GenericAssetType(name="ShiftableLoad") + + db.session.add(shiftable_load_asset_type) + + shiftable_asset = GenericAsset( + name="Test Asset", generic_asset_type=shiftable_load_asset_type + ) + + db.session.add(shiftable_asset) + + power_sensor = Sensor( + "power", + generic_asset=shiftable_asset, + event_resolution=timedelta(hours=1), + attributes={ + "custom-scheduler": { + "class": "ShiftableLoadScheduler", + "module": "flexmeasures.data.models.planning.shiftable_load", + } + }, + unit="MW", + ) + + db.session.add(power_sensor) + + db.session.commit() + + yield power_sensor.id diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index cb924b47c..7cbbba139 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -211,3 +211,47 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): ) assert len(stored_report) == 95 + + +@pytest.mark.skip_github +@pytest.mark.parametrize("load_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) +def test_add_shiftable( + app, db, shiftable_load_power_sensor, add_market_prices, load_type +): + """ + Schedule a 4h of consumption block at a constant power of 400kW in a day using + the three shiftable policies: inflexible, shiftable and breakable. + """ + + from flexmeasures.cli.data_add import add_schedule_shiftable_load + + epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() + + shiftable_load_power_sensor_id = shiftable_load_power_sensor + + cli_input_params = { + "sensor-id": shiftable_load_power_sensor_id, + "start": "2015-01-02T00:00:00+01:00", + "duration": "PT24H", + "load-duration": "PT4H", + "load-power": "0.4MW", + "load-type": load_type, + "consumption-price-sensor": epex_da.id, + } + + cli_input = to_flags(cli_input_params) + runner = app.test_cli_runner() + + # call command + result = runner.invoke(add_schedule_shiftable_load, cli_input) + + print(result) + + assert result.exit_code == 0 # run command without errors + + shiftable_load_power_sensor = Sensor.query.get(shiftable_load_power_sensor_id) + + schedule = shiftable_load_power_sensor.search_beliefs() + assert (schedule == -0.4).event_value.sum() == 4 + + # schedule = shiftable_load_power_sensor From 850dd511c08fc0a0898082445f5d3c4a6fa8a90a Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 21 Jul 2023 14:49:23 +0200 Subject: [PATCH 12/48] docs: add changelog Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index eaaf66075..7caf098e9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] +* Add `ShiftableLoadScheduler` class, which optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_] Bugfixes ----------- @@ -28,8 +29,6 @@ Infrastructure / Support * The endpoint `[POST] /health/ready `_ returns the status of the Redis connection, if configured [see `PR #699 `_] * Document the `device_scheduler` linear program [see `PR #764 `_]. -/api/v3_0/health/ready - v0.14.1 | June 26, 2023 ============================ From b099e254003a1d175b5b209c1fdfc441a4f3b014 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 21 Jul 2023 15:18:48 +0200 Subject: [PATCH 13/48] refactor: move TimeIntervalSchema to data.schemas.time Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/__init__.py | 2 +- .../data/schemas/scheduling/shiftable_load.py | 12 +++++------ flexmeasures/data/schemas/times.py | 21 ++++++++++++++++++- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index 64be51f77..5965fb501 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -6,4 +6,4 @@ from .generic_assets import GenericAssetIdField as AssetIdField # noqa F401 from .sensors import SensorIdField # noqa F401 from .sources import DataSourceIdField as SourceIdField # noqa F401 -from .times import AwareDateTimeField, DurationField # noqa F401 +from .times import AwareDateTimeField, DurationField, TimeIntervalField # noqa F401 diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/shiftable_load.py index d189a2583..e78437f93 100644 --- a/flexmeasures/data/schemas/scheduling/shiftable_load.py +++ b/flexmeasures/data/schemas/scheduling/shiftable_load.py @@ -12,7 +12,10 @@ ) from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField +from flexmeasures.data.schemas.times import ( + DurationField, + TimeIntervalSchema, +) from flexmeasures.data.schemas.sensors import SensorIdField @@ -31,13 +34,8 @@ class OptimizationSense(Enum): MIN = "MIN" -class TimeIntervalSchema(Schema): - start = AwareDateTimeField(required=True) - duration = DurationField(required=True) - - class ShiftableLoadFlexModelSchema(Schema): - cost_sensor = SensorIdField(data_key="cost-sensor") + consumption_price_sensor = SensorIdField(data_key="consumption-price-sensor") duration = DurationField(required=True) power = fields.Float(required=True) diff --git a/flexmeasures/data/schemas/times.py b/flexmeasures/data/schemas/times.py index f2f931356..29f6df7c3 100644 --- a/flexmeasures/data/schemas/times.py +++ b/flexmeasures/data/schemas/times.py @@ -1,9 +1,11 @@ from __future__ import annotations +import json from datetime import datetime, timedelta from flask import current_app -from marshmallow import fields +from marshmallow import fields, Schema +from marshmallow.exceptions import ValidationError import isodate from isodate.isoerror import ISO8601Error import pandas as pd @@ -84,3 +86,20 @@ def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: """ value = value.replace(" ", "+") return fields.AwareDateTime._deserialize(self, value, attr, obj, **kwargs) + + +class TimeIntervalSchema(Schema): + start = AwareDateTimeField(required=True) + duration = DurationField(required=True) + + +class TimeIntervalField(MarshmallowClickMixin, fields.Dict): + """Field that de-serializes to a TimeInverval defined with start and duration.""" + + def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: + try: + v = json.loads(value) + except json.JSONDecodeError: + raise ValidationError() + + return TimeIntervalSchema().load(v) From bf8c20d5486aad2e3cf439e3f3afa9e31b4d2c8e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 21 Jul 2023 15:20:00 +0200 Subject: [PATCH 14/48] refactor: rename cost_sensor to consumption_price_sensor Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 14 +++++++++----- .../models/planning/tests/test_shiftable_load.py | 10 +++++----- flexmeasures/data/schemas/tests/test_scheduling.py | 6 +++--- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 78b171cb3..2caa428e4 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -37,7 +37,7 @@ def compute(self) -> pd.Series | None: Parameters ========== - cost_sensor: it defines the utility (economic, environmental, ) in each + consumption_price_sensor: it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. power: nominal power of the load. duration: time that the load lasts. @@ -58,7 +58,9 @@ def compute(self) -> pd.Series | None: belief_time = self.belief_time sensor = self.sensor - cost_sensor: Sensor = self.flex_model.get("cost_sensor") + consumption_price_sensor: Sensor = self.flex_model.get( + "consumption_price_sensor" + ) duration: timedelta = self.flex_model.get("duration") power = self.flex_model.get("power") optimization_sense = self.flex_model.get("optimization_sense") @@ -66,7 +68,7 @@ def compute(self) -> pd.Series | None: time_restrictions = self.flex_model.get("time_restrictions") # get cost data - cost = cost_sensor.search_beliefs( + cost = consumption_price_sensor.search_beliefs( event_starts_after=start, event_ends_before=end, resolution=resolution, @@ -89,11 +91,13 @@ def compute(self) -> pd.Series | None: ) # optimize schedule for tomorrow. We can fill len(schedule) rows, at most - rows_to_fill = min(ceil(duration / cost_sensor.event_resolution), len(schedule)) + rows_to_fill = min( + ceil(duration / consumption_price_sensor.event_resolution), len(schedule) + ) # convert power to energy using the resolution of the sensor. # e.g. resolution=15min, power=1kW -> energy=250W - energy = power * cost_sensor.event_resolution / timedelta(hours=1) + energy = power * consumption_price_sensor.event_resolution / timedelta(hours=1) if rows_to_fill > len(schedule): raise ValueError( diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_shiftable_load.py index 704cc52e7..63027deb4 100644 --- a/flexmeasures/data/models/planning/tests/test_shiftable_load.py +++ b/flexmeasures/data/models/planning/tests/test_shiftable_load.py @@ -28,7 +28,7 @@ def test_shiftable_scheduler( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "cost-sensor": epex_da.id, + "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": load_type, "power": 4, @@ -68,7 +68,7 @@ def test_duration_exceeds_planning_window( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "cost-sensor": epex_da.id, + "consumption-price-sensor": epex_da.id, "duration": "PT48H", "load-type": load_type, "power": 4, @@ -100,7 +100,7 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa # time parameters flex_model = { - "cost-sensor": epex_da.id, + "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": "SHIFTABLE", "power": 4, @@ -147,7 +147,7 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa # time parameters flex_model = { - "cost-sensor": epex_da.id, + "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": "BREAKABLE", "power": 4, @@ -199,7 +199,7 @@ def test_impossible_schedules( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "cost-sensor": epex_da.id, + "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": load_type, "power": 4, diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 52d94ac63..04c887918 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -19,7 +19,7 @@ def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): shiftable_load_flex_model = schema.load( { - "cost-sensor": sensor1.id, + "consumption-price-sensor": sensor1.id, "duration": "PT4H", "power": 30.0, "time-restrictions": [ @@ -45,7 +45,7 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): shiftable_load_flex_model = schema.load( { - "cost-sensor": sensor1.id, + "consumption-price-sensor": sensor1.id, "duration": "PT4H", "power": 30.0, "time-restrictions": [ @@ -68,7 +68,7 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): { "duration": "PT4H", "power": 30.0, - "cost-sensor": sensor1.id, + "consumption-price-sensor": sensor1.id, "time-restrictions": [ {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} ], From e820ffc7d9cee7ab706d58045289c15e40b48d51 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 21 Jul 2023 16:03:25 +0200 Subject: [PATCH 15/48] feat: add forbid option Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 21 +++++++++++++++++++-- flexmeasures/cli/tests/test_data_add.py | 1 + 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 8b4515f59..cbc4a0dbe 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -5,7 +5,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Type +from typing import Type, List import json from pathlib import Path from io import TextIOBase @@ -51,7 +51,9 @@ LatitudeField, LongitudeField, SensorIdField, + TimeIntervalField, ) +from flexmeasures.data.schemas.times import TimeIntervalSchema from flexmeasures.data.schemas.scheduling.storage import EfficiencyField from flexmeasures.data.schemas.sensors import SensorSchema from flexmeasures.data.schemas.units import QuantityField @@ -1210,6 +1212,15 @@ def add_schedule_for_storage( required=True, help="Constant power of the load during the activation period.", ) +@click.option( + "--forbid", + type=TimeIntervalField(), + multiple=True, + required=False, + help="Add time restrictions to the optimization, where the load will not be scheduled into." + 'Use the following format to define the restrictions: `{"start":, "duration":}`' + "This options allows to define multiple time restrictions by using the --forbid for different periods.", +) @click.option( "--as-job", is_flag=True, @@ -1224,6 +1235,7 @@ def add_schedule_shiftable_load( load_duration: timedelta, load_type: str, load_power: ur.Quantity, + forbid: List | None = None, as_job: bool = False, ): """Create a new schedule for a shiftable asset. @@ -1232,6 +1244,10 @@ def add_schedule_shiftable_load( - Only supports consumption blocks. - Not taking into account grid constraints or other loads. """ + + if forbid is None: + forbid = [] + # Parse input and required sensor attributes if not power_sensor.measures_power: click.secho( @@ -1250,10 +1266,11 @@ def add_schedule_shiftable_load( belief_time=server_now(), resolution=power_sensor.event_resolution, flex_model={ - "cost-sensor": consumption_price_sensor.id, + "consumption-price-sensor": consumption_price_sensor.id, "duration": pd.Timedelta(load_duration).isoformat(), "load-type": load_type, "power": load_power, + "time-restrictions": [TimeIntervalSchema().dump(f) for f in forbid], }, ) diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 7cbbba139..a276ee305 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -237,6 +237,7 @@ def test_add_shiftable( "load-power": "0.4MW", "load-type": load_type, "consumption-price-sensor": epex_da.id, + "forbid": '{"start" : "2015-01-02T00:00:00+01:00", "duration" : "PT2H"}', } cli_input = to_flags(cli_input_params) From 405ca40cc20e7e821f758d607ee3d467547983e8 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 21 Jul 2023 16:13:47 +0200 Subject: [PATCH 16/48] docs: add attribute description Signed-off-by: Victor Garcia Reolid --- documentation/api/notation.rst | 8 +++++++- flexmeasures/data/models/planning/shiftable_load.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index ff1f44b22..895204a56 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -204,7 +204,13 @@ Here are the three types of flexibility models you can expect to be built-in: 2) For **shiftable processes** - .. todo:: A simple and proven algorithm exists, but is awaiting proper integration into FlexMeasures, see `PR 729 `_. + - ``consumption_price_sensor``: it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. + - ``power``: nominal power of the load. + - ``duration``: time that the load last. + - ``optimization_sense``: objective of the scheduler, to maximize or minimize. + - ``time_restrictions``: time periods in which the load cannot be schedule to. + - ``load_type``: Inflexible, Breakable or Shiftable. + 3) For **buffer devices** (e.g. thermal energy storage systems connected to heat pumps), use the same flexibility parameters described above for storage devices. Here are some tips to model a buffer with these parameters: diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 2caa428e4..570aa964c 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -40,7 +40,7 @@ def compute(self) -> pd.Series | None: consumption_price_sensor: it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. power: nominal power of the load. - duration: time that the load lasts. + duration: time that the load last. optimization_sense: objective of the scheduler, to maximize or minimize. time_restrictions: time periods in which the load cannot be schedule to. From c20669c01065a9473cee33c3cff1510d3ee966c5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 13:45:41 +0200 Subject: [PATCH 17/48] address change requests Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 98 +++++++++++++------ .../planning/tests/test_shiftable_load.py | 3 +- .../data/schemas/scheduling/shiftable_load.py | 12 ++- flexmeasures/data/schemas/times.py | 2 +- flexmeasures/data/services/scheduling.py | 3 + 5 files changed, 85 insertions(+), 33 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 570aa964c..5a32cd7b4 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -24,8 +24,11 @@ class ShiftableLoadScheduler(Scheduler): __author__ = "Seita" def compute(self) -> pd.Series | None: - """Schedule a load, defined as a `power` and a `duration`, within the specified time window. - For example, this scheduler can plan the start of a process that lasts 5h and requires a power of 10kW. + """Schedule a fix load, defined as a `power` and a `duration`, within the specified time window. + To schedule a battery, please, refer to the StorageScheduler. + + For example, this scheduler can plan the start of a process of type `Shiftable` that lasts 5h and requires a power of 10kW. + In that case, the scheduler will find the best (as to minimize/maximize the cost) hour to start the process. This scheduler supports three types of `load_types`: - Inflexible: this load requires to be scheduled as soon as possible. @@ -90,38 +93,25 @@ def compute(self) -> pd.Series | None: name="event_value", ) - # optimize schedule for tomorrow. We can fill len(schedule) rows, at most - rows_to_fill = min( - ceil(duration / consumption_price_sensor.event_resolution), len(schedule) - ) - # convert power to energy using the resolution of the sensor. # e.g. resolution=15min, power=1kW -> energy=250W energy = power * consumption_price_sensor.event_resolution / timedelta(hours=1) - if rows_to_fill > len(schedule): - raise ValueError( - f"Duration of the period exceeds the schedule window. The resulting schedule will be trimmed to fit the planning window ({start}, {end})." - ) + # we can fill duration/resolution rows or, if the duration is larger than the schedule + # window, fill the entire window. + rows_to_fill = min( + ceil(duration / consumption_price_sensor.event_resolution), len(schedule) + ) - # check if the time_restrictions allow for a load of the duration provided - if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: - # get start time instants that are not feasible, i.e. some time during the ON period goes through - # a time restriction interval - time_restrictions = ( - time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) - ) - time_restrictions = (time_restrictions == 1) | time_restrictions.isna() + if rows_to_fill == len(schedule): + schedule[:] = energy + return schedule - if time_restrictions.sum() == len(time_restrictions): - raise ValueError( - "Cannot allocate a block of time {duration} given the time restrictions provided." - ) - else: # LoadType.BREAKABLE - if (~time_restrictions).sum() < rows_to_fill: - raise ValueError( - "Cannot allocate a block of time {duration} given the time restrictions provided." - ) + time_restrictions = ( + self.block_invalid_starting_times_for_whole_process_scheduling( + load_type, time_restrictions, duration, rows_to_fill + ) + ) # create schedule if load_type == LoadType.INFLEXIBLE: @@ -149,6 +139,56 @@ def compute(self) -> pd.Series | None: return schedule.tz_convert(self.start.tzinfo) + def block_invalid_starting_times_for_whole_process_scheduling( + self, + load_type: LoadType, + time_restrictions: pd.Series, + duration: timedelta, + rows_to_fill: int, + ) -> pd.Series: + """Blocks time periods where the load cannot be schedule into, making + sure no other time restrictions runs in the middle of the activation of the load + + More technically, this function applying an erosion of the time_restrictions array with a block of length duration. + + Then, the condition if time_restrictions.sum() == len(time_restrictions):, makes sure that at least we have a spot to place the load. + + For example: + + time_restriction = [1 0 0 1 1 1 0 0 1 0] + + # applying a dilation with duration = 2 + time_restriction = [1 0 1 1 1 1 0 1 1 1] + + We can only fit a block of duration = 2 in the positions 1 and 6. sum(time_restrictions) == 8, + while the len(time_restriction) == 10, which means we have 10-8=2 positions. + + :param load_type: INFLEXIBLE, SHIFTABLE or BREAKABLE + :param time_restrictions: boolean time series indicating time periods in which the load cannot be scheduled. + :param duration: (datetime) duration of the length + :param rows_to_fill: (int) time periods that the load lasts + :return: filtered time restrictions + """ + + if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: + # get start time instants that are not feasible, i.e. some time during the ON period goes through + # a time restriction interval + time_restrictions = ( + time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) + ) + time_restrictions = (time_restrictions == 1) | time_restrictions.isna() + + if time_restrictions.sum() == len(time_restrictions): + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." + ) + else: # LoadType.BREAKABLE + if (~time_restrictions).sum() < rows_to_fill: + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." + ) + return time_restrictions + def compute_inflexible( self, schedule: pd.Series, @@ -203,7 +243,7 @@ def compute_shiftable( else: start = block_cost[~time_restrictions].idxmax() - start = start.event_value + start = start.iloc[0] schedule.loc[start : start + self.resolution * (rows_to_fill - 1)] = energy diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_shiftable_load.py index 63027deb4..badbf4ae1 100644 --- a/flexmeasures/data/models/planning/tests/test_shiftable_load.py +++ b/flexmeasures/data/models/planning/tests/test_shiftable_load.py @@ -138,7 +138,8 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa """ Test breakable load_type of ShiftableLoadScheduler by introducing four 1-hour restrictions interspaced by 1 hour. The equivalent mask would be the following: [0,...,0,1,0,1,0,1,0,1,0, ...,0]. - This makes the schedule choose time periods between. + Trying to get the best prices (between 9am and 4pm), his makes the schedule choose time periods between + the time restrictions. """ # get the sensors from the database diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/shiftable_load.py index e78437f93..fdddf0979 100644 --- a/flexmeasures/data/schemas/scheduling/shiftable_load.py +++ b/flexmeasures/data/schemas/scheduling/shiftable_load.py @@ -35,18 +35,23 @@ class OptimizationSense(Enum): class ShiftableLoadFlexModelSchema(Schema): + # it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. consumption_price_sensor = SensorIdField(data_key="consumption-price-sensor") + # time that the load last. duration = DurationField(required=True) + # nominal power of the load. power = fields.Float(required=True) - + # policy to schedule a load: INFLEXIBLE, SHIFTABLE, BREAKABLE load_type = fields.Enum( LoadType, load_default=LoadType.INFLEXIBLE, data_key="load-type" ) + # time_restrictions will be turned into a Series with Boolean values (where True means restricted for scheduling). time_restrictions = fields.List( fields.Nested(TimeIntervalSchema()), data_key="time-restrictions", load_default=[], ) + # objective of the scheduler, to maximize or minimize. optimization_sense = fields.Enum( OptimizationSense, load_default=OptimizationSense.MIN, @@ -102,12 +107,15 @@ def post_load_time_restrictions(self, data: dict, **kwargs) -> dict: @pre_load def pre_load_load_type(self, data: dict, **kwargs) -> dict: """Fallback mechanism for the load_type variable. If not found in data, - it tries to find it in among the sensor attributes and, if it's not found + it tries to find it in among the sensor or asset attributes and, if it's not found there either, it defaults to "INFLEXIBLE". """ if "load-type" not in data or data["load-type"] is None: load_type = self.sensor.get_attribute("load_type") + if load_type is None: + load_type = self.sensor.generic_asset.get_attribute("load_type") + if load_type is None: load_type = "INFLEXIBLE" diff --git a/flexmeasures/data/schemas/times.py b/flexmeasures/data/schemas/times.py index 29f6df7c3..b56d70352 100644 --- a/flexmeasures/data/schemas/times.py +++ b/flexmeasures/data/schemas/times.py @@ -96,7 +96,7 @@ class TimeIntervalSchema(Schema): class TimeIntervalField(MarshmallowClickMixin, fields.Dict): """Field that de-serializes to a TimeInverval defined with start and duration.""" - def _deserialize(self, value: str, attr, obj, **kwargs) -> datetime: + def _deserialize(self, value: str, attr, obj, **kwargs) -> dict: try: v = json.loads(value) except json.JSONDecodeError: diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 526085874..6b077e9bd 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -19,6 +19,7 @@ from flexmeasures.data import db from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.models.planning.storage import StorageScheduler +from flexmeasures.data.models.planning.shiftable_load import ShiftableLoadScheduler from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.utils import get_data_source, save_to_db @@ -194,6 +195,8 @@ def find_scheduler_class(sensor: Sensor) -> type: "two-way_evse", ): scheduler_class = StorageScheduler + elif sensor.generic_asset.generic_asset_type.name in ("LoadType"): + scheduler_class = ShiftableLoadScheduler else: raise ValueError( "Scheduling is not (yet) supported for asset type %s." From 98d4a164ea999a954440d9d792f53bc7d33e0f7e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 14:12:18 +0200 Subject: [PATCH 18/48] use consumption_price_sensor from flex_context Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 2 +- .../planning/tests/test_shiftable_load.py | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 5a32cd7b4..9697e9d9b 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -61,7 +61,7 @@ def compute(self) -> pd.Series | None: belief_time = self.belief_time sensor = self.sensor - consumption_price_sensor: Sensor = self.flex_model.get( + consumption_price_sensor: Sensor = self.flex_context.get( "consumption_price_sensor" ) duration: timedelta = self.flex_model.get("duration") diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_shiftable_load.py index badbf4ae1..156a25c3d 100644 --- a/flexmeasures/data/models/planning/tests/test_shiftable_load.py +++ b/flexmeasures/data/models/planning/tests/test_shiftable_load.py @@ -28,18 +28,22 @@ def test_shiftable_scheduler( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": load_type, "power": 4, } + flex_context = { + "consumption-price-sensor": epex_da.id, + } + scheduler = ShiftableLoadScheduler( shiftable_load, start, end, resolution, flex_model=flex_model, + flex_context=flex_context, ) schedule = scheduler.compute() @@ -68,18 +72,22 @@ def test_duration_exceeds_planning_window( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "consumption-price-sensor": epex_da.id, "duration": "PT48H", "load-type": load_type, "power": 4, } + flex_context = { + "consumption-price-sensor": epex_da.id, + } + scheduler = ShiftableLoadScheduler( shiftable_load, start, end, resolution, flex_model=flex_model, + flex_context=flex_context, ) schedule = scheduler.compute() @@ -100,7 +108,6 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa # time parameters flex_model = { - "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": "SHIFTABLE", "power": 4, @@ -108,6 +115,9 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa {"start": "2015-01-02T08:00:00+01:00", "duration": "PT2H"} ], } + flex_context = { + "consumption-price-sensor": epex_da.id, + } scheduler = ShiftableLoadScheduler( shiftable_load, @@ -115,6 +125,7 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa end, resolution, flex_model=flex_model, + flex_context=flex_context, ) schedule = scheduler.compute() @@ -148,7 +159,6 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa # time parameters flex_model = { - "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": "BREAKABLE", "power": 4, @@ -160,12 +170,17 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa ], } + flex_context = { + "consumption-price-sensor": epex_da.id, + } + scheduler = ShiftableLoadScheduler( shiftable_load, start, end, resolution, flex_model=flex_model, + flex_context=flex_context, ) schedule = scheduler.compute() @@ -200,12 +215,14 @@ def test_impossible_schedules( epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() flex_model = { - "consumption-price-sensor": epex_da.id, "duration": "PT4H", "load-type": load_type, "power": 4, "time-restrictions": time_restrictions, } + flex_context = { + "consumption-price-sensor": epex_da.id, + } scheduler = ShiftableLoadScheduler( shiftable_load, @@ -213,6 +230,7 @@ def test_impossible_schedules( end, resolution, flex_model=flex_model, + flex_context=flex_context, ) with pytest.raises(ValueError): From c4e87f6855c29228feb1e3007ec45f68a02508aa Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 14:15:42 +0200 Subject: [PATCH 19/48] communicate consumption_price_sensor through the flex_context Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 8 +++++--- flexmeasures/cli/tests/test_data_add.py | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index cbc4a0dbe..2ff70899b 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1160,7 +1160,7 @@ def add_schedule_for_storage( click.secho("New schedule is stored.", **MsgStyle.SUCCESS) -@create_schedule.command("shiftable") +@create_schedule.command("for-shiftable") @with_appcontext @click.option( "--sensor-id", @@ -1202,7 +1202,7 @@ def add_schedule_for_storage( "load_type", type=click.Choice(["INFLEXIBLE", "BREAKABLE", "SHIFTABLE"], case_sensitive=False), required=False, - default="INFLEXIBLE", + default="SHIFTABLE", help="Load shift policy type: INFLEXIBLE, BREAKABLE or SHIFTABLE.", ) @click.option( @@ -1266,12 +1266,14 @@ def add_schedule_shiftable_load( belief_time=server_now(), resolution=power_sensor.event_resolution, flex_model={ - "consumption-price-sensor": consumption_price_sensor.id, "duration": pd.Timedelta(load_duration).isoformat(), "load-type": load_type, "power": load_power, "time-restrictions": [TimeIntervalSchema().dump(f) for f in forbid], }, + flex_context={ + "consumption-price-sensor": consumption_price_sensor.id, + }, ) if as_job: diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index a276ee305..4ef305630 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -254,5 +254,3 @@ def test_add_shiftable( schedule = shiftable_load_power_sensor.search_beliefs() assert (schedule == -0.4).event_value.sum() == 4 - - # schedule = shiftable_load_power_sensor From de5e3cc91d6bd46c7b899ab3d8884f1d54204859 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 14:25:45 +0200 Subject: [PATCH 20/48] add clarifying comments Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 6 +++--- flexmeasures/cli/tests/test_data_add.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 029e34915..f613ab0d9 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -102,11 +102,11 @@ def reporter_config_raw(app, db, setup_dummy_data): @pytest.fixture(scope="module") def shiftable_load_power_sensor(db, app): """ - Create an asset with two sensors (1 and 2), and add the same set of 200 beliefs with an hourly resolution to each of them. - Return the two sensors and a result sensor (which has no data). + Create an asset of type "LoadType" and a power sensor to hold the result of + the scheduler. """ - shiftable_load_asset_type = GenericAssetType(name="ShiftableLoad") + shiftable_load_asset_type = GenericAssetType(name="LoadType") db.session.add(shiftable_load_asset_type) diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 4ef305630..9b14b0564 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -253,4 +253,6 @@ def test_add_shiftable( shiftable_load_power_sensor = Sensor.query.get(shiftable_load_power_sensor_id) schedule = shiftable_load_power_sensor.search_beliefs() + # check if the schedule is not empty more detailed testing can be found + # in data/models/planning/tests/test_shiftable_loads.py. assert (schedule == -0.4).event_value.sum() == 4 From b991863b40072d38dc9b3cf9fab9f4a78a868f2b Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 15:09:38 +0200 Subject: [PATCH 21/48] making block_invalid_starting_times_for_whole_process_scheduling work only when optimizing a INFLEXIBLE or SHIFTABLE load type. Signed-off-by: Victor Garcia Reolid --- .../data/models/planning/shiftable_load.py | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 9697e9d9b..0a1350cd8 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -107,15 +107,23 @@ def compute(self) -> pd.Series | None: schedule[:] = energy return schedule - time_restrictions = ( - self.block_invalid_starting_times_for_whole_process_scheduling( - load_type, time_restrictions, duration, rows_to_fill + if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: + start_time_restrictions = ( + self.block_invalid_starting_times_for_whole_process_scheduling( + load_type, time_restrictions, duration, rows_to_fill + ) ) - ) + else: # LoadType.BREAKABLE + if (~time_restrictions).sum() < rows_to_fill: + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." + ) # create schedule if load_type == LoadType.INFLEXIBLE: - self.compute_inflexible(schedule, time_restrictions, rows_to_fill, energy) + self.compute_inflexible( + schedule, start_time_restrictions, rows_to_fill, energy + ) elif load_type == LoadType.BREAKABLE: self.compute_breakable( schedule, @@ -129,7 +137,7 @@ def compute(self) -> pd.Series | None: self.compute_shiftable( schedule, optimization_sense, - time_restrictions, + start_time_restrictions, cost, rows_to_fill, energy, @@ -160,7 +168,7 @@ def block_invalid_starting_times_for_whole_process_scheduling( # applying a dilation with duration = 2 time_restriction = [1 0 1 1 1 1 0 1 1 1] - We can only fit a block of duration = 2 in the positions 1 and 6. sum(time_restrictions) == 8, + We can only fit a block of duration = 2 in the positions 1 and 6. sum(start_time_restrictions) == 8, while the len(time_restriction) == 10, which means we have 10-8=2 positions. :param load_type: INFLEXIBLE, SHIFTABLE or BREAKABLE @@ -170,24 +178,21 @@ def block_invalid_starting_times_for_whole_process_scheduling( :return: filtered time restrictions """ - if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: - # get start time instants that are not feasible, i.e. some time during the ON period goes through - # a time restriction interval - time_restrictions = ( - time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) + # get start time instants that are not feasible, i.e. some time during the ON period goes through + # a time restriction interval + start_time_restrictions = ( + time_restrictions.rolling(duration).max().shift(-rows_to_fill + 1) + ) + start_time_restrictions = ( + start_time_restrictions == 1 + ) | start_time_restrictions.isna() + + if (~start_time_restrictions).sum() == 0: + raise ValueError( + "Cannot allocate a block of time {duration} given the time restrictions provided." ) - time_restrictions = (time_restrictions == 1) | time_restrictions.isna() - if time_restrictions.sum() == len(time_restrictions): - raise ValueError( - "Cannot allocate a block of time {duration} given the time restrictions provided." - ) - else: # LoadType.BREAKABLE - if (~time_restrictions).sum() < rows_to_fill: - raise ValueError( - "Cannot allocate a block of time {duration} given the time restrictions provided." - ) - return time_restrictions + return start_time_restrictions def compute_inflexible( self, From 68229580e2a0295e5fd5c93de0a8a6d742fcdee1 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 25 Jul 2023 15:12:12 +0200 Subject: [PATCH 22/48] remove consumption_price_sensor from flex_model Signed-off-by: Victor Garcia Reolid --- documentation/api/notation.rst | 1 - flexmeasures/data/schemas/scheduling/shiftable_load.py | 4 ---- flexmeasures/data/schemas/tests/test_scheduling.py | 3 --- 3 files changed, 8 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 895204a56..b00580715 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -204,7 +204,6 @@ Here are the three types of flexibility models you can expect to be built-in: 2) For **shiftable processes** - - ``consumption_price_sensor``: it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. - ``power``: nominal power of the load. - ``duration``: time that the load last. - ``optimization_sense``: objective of the scheduler, to maximize or minimize. diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/shiftable_load.py index fdddf0979..85675fd7a 100644 --- a/flexmeasures/data/schemas/scheduling/shiftable_load.py +++ b/flexmeasures/data/schemas/scheduling/shiftable_load.py @@ -18,8 +18,6 @@ ) -from flexmeasures.data.schemas.sensors import SensorIdField - from enum import Enum @@ -35,8 +33,6 @@ class OptimizationSense(Enum): class ShiftableLoadFlexModelSchema(Schema): - # it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. - consumption_price_sensor = SensorIdField(data_key="consumption-price-sensor") # time that the load last. duration = DurationField(required=True) # nominal power of the load. diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 04c887918..508c8f071 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -19,7 +19,6 @@ def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): shiftable_load_flex_model = schema.load( { - "consumption-price-sensor": sensor1.id, "duration": "PT4H", "power": 30.0, "time-restrictions": [ @@ -45,7 +44,6 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): shiftable_load_flex_model = schema.load( { - "consumption-price-sensor": sensor1.id, "duration": "PT4H", "power": 30.0, "time-restrictions": [ @@ -68,7 +66,6 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): { "duration": "PT4H", "power": 30.0, - "consumption-price-sensor": sensor1.id, "time-restrictions": [ {"start": "2023-01-01T00:00:00+00:00", "duration": "PT3H"} ], From 7dc318785b2939d4e1ced5674132b441cf22e0d5 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 12:26:19 +0200 Subject: [PATCH 23/48] add changelog entry Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 46802ac92..a41a9486b 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] -* Add `ShiftableLoadScheduler` class, which optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_] +* Add `ShiftableLoadScheduler` class and CLI command `flexmeasures add schedule for-shiftable-load` to optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_ and `PR #768 `_] Bugfixes ----------- From 51de43e1bed04e5f54b502f2bf8a01fbc1286ca6 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 12:28:05 +0200 Subject: [PATCH 24/48] CLI changelog entry Signed-off-by: Victor Garcia Reolid --- documentation/cli/change_log.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 2787c7e51..636db1927 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -8,6 +8,8 @@ since v0.15.0 | July XX, 2023 ================================= * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times. +* Add ``flexmeasures add schedule for-shiftable-load`` to create a new shiftable load schedule for a given power sensor. + since v0.14.1 | June XX, 2023 ================================= From f35fdf73558e745f881917d652c3dd6dc10d693d Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 12:30:11 +0200 Subject: [PATCH 25/48] add flexmeasures add schedule for-shiftable-load to commands list Signed-off-by: Victor Garcia Reolid --- documentation/cli/commands.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index ee31d2d54..02f76bea8 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -36,6 +36,7 @@ of which some are referred to in this documentation. ``flexmeasures add source`` Add a new data source. ``flexmeasures add forecasts`` Create forecasts. ``flexmeasures add schedule for-storage`` Create a charging schedule for a storage asset. +``flexmeasures add schedule for-shiftable-load`` Create a schedule for a shiftable load. ``flexmeasures add holidays`` Add holiday annotations to accounts and/or assets. ``flexmeasures add annotation`` Add annotation to accounts, assets and/or sensors. ``flexmeasures add toy-account`` Create a toy account, for tutorials and trying things. From 05c91e3abf6382c948fd2756e79b27a15dfbd0ae Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 12:36:46 +0200 Subject: [PATCH 26/48] add docstring for fixture shiftable_load_power_sensor Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index f613ab0d9..43ad86b82 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -104,6 +104,7 @@ def shiftable_load_power_sensor(db, app): """ Create an asset of type "LoadType" and a power sensor to hold the result of the scheduler. + """ shiftable_load_asset_type = GenericAssetType(name="LoadType") From 5938f50655b4b16b0916ad4c9582c7ef7c81abdf Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 12:41:15 +0200 Subject: [PATCH 27/48] simplify schedule sensor attributes Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 2 +- flexmeasures/cli/tests/conftest.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 2ff70899b..74dbdda23 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1210,7 +1210,7 @@ def add_schedule_for_storage( "load_power", type=ur.Quantity, required=True, - help="Constant power of the load during the activation period.", + help="Constant power of the load during the activation period, e.g. 4kW.", ) @click.option( "--forbid", diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 43ad86b82..1ac161344 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -121,12 +121,6 @@ def shiftable_load_power_sensor(db, app): "power", generic_asset=shiftable_asset, event_resolution=timedelta(hours=1), - attributes={ - "custom-scheduler": { - "class": "ShiftableLoadScheduler", - "module": "flexmeasures.data.models.planning.shiftable_load", - } - }, unit="MW", ) From a673c48f2523a1ab6cfc0304f0201a615a662041 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 13:01:39 +0200 Subject: [PATCH 28/48] fix flexmeasures add schedule for-shiftable command name Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- documentation/cli/change_log.rst | 2 +- documentation/cli/commands.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index a41a9486b..a5754411e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] -* Add `ShiftableLoadScheduler` class and CLI command `flexmeasures add schedule for-shiftable-load` to optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_ and `PR #768 `_] +* Add `ShiftableLoadScheduler` class and CLI command `flexmeasures add schedule for-shiftable` to optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_ and `PR #768 `_] Bugfixes ----------- diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 636db1927..05304c6eb 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -8,7 +8,7 @@ since v0.15.0 | July XX, 2023 ================================= * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times. -* Add ``flexmeasures add schedule for-shiftable-load`` to create a new shiftable load schedule for a given power sensor. +* Add ``flexmeasures add schedule for-shiftable`` to create a new shiftable load schedule for a given power sensor. since v0.14.1 | June XX, 2023 diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 02f76bea8..6ac8dcdc8 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -36,7 +36,7 @@ of which some are referred to in this documentation. ``flexmeasures add source`` Add a new data source. ``flexmeasures add forecasts`` Create forecasts. ``flexmeasures add schedule for-storage`` Create a charging schedule for a storage asset. -``flexmeasures add schedule for-shiftable-load`` Create a schedule for a shiftable load. +``flexmeasures add schedule for-shiftable`` Create a schedule for a shiftable load. ``flexmeasures add holidays`` Add holiday annotations to accounts and/or assets. ``flexmeasures add annotation`` Add annotation to accounts, assets and/or sensors. ``flexmeasures add toy-account`` Create a toy account, for tutorials and trying things. From b548ee5126a6732ac60897f0b9c891ea32dfb6be Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 13:19:50 +0200 Subject: [PATCH 29/48] fix: wrong resolution Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/shiftable_load.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/shiftable_load.py index 0a1350cd8..999ea0bbe 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/shiftable_load.py @@ -95,13 +95,11 @@ def compute(self) -> pd.Series | None: # convert power to energy using the resolution of the sensor. # e.g. resolution=15min, power=1kW -> energy=250W - energy = power * consumption_price_sensor.event_resolution / timedelta(hours=1) + energy = power * self.resolution / timedelta(hours=1) # we can fill duration/resolution rows or, if the duration is larger than the schedule # window, fill the entire window. - rows_to_fill = min( - ceil(duration / consumption_price_sensor.event_resolution), len(schedule) - ) + rows_to_fill = min(ceil(duration / self.resolution), len(schedule)) if rows_to_fill == len(schedule): schedule[:] = energy From 1529a9197b5b81fcf73a8640def66d027f78184c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 23:07:28 +0200 Subject: [PATCH 30/48] refactor: rename shiftable_load to process Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- .../{shiftable_load.py => process.py} | 62 +++++++++---------- .../data/models/planning/tests/conftest.py | 12 ++-- ...test_shiftable_load.py => test_process.py} | 58 +++++++++-------- .../{shiftable_load.py => process.py} | 34 +++++----- .../data/schemas/tests/test_scheduling.py | 30 ++++----- flexmeasures/data/services/scheduling.py | 6 +- 7 files changed, 101 insertions(+), 103 deletions(-) rename flexmeasures/data/models/planning/{shiftable_load.py => process.py} (82%) rename flexmeasures/data/models/planning/tests/{test_shiftable_load.py => test_process.py} (80%) rename flexmeasures/data/schemas/scheduling/{shiftable_load.py => process.py} (75%) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 46802ac92..81a45ed96 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] -* Add `ShiftableLoadScheduler` class, which optimizes loads activation using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_] +* Add `ProcessScheduler` class, which optimizes the starting time of processes using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_] Bugfixes ----------- diff --git a/flexmeasures/data/models/planning/shiftable_load.py b/flexmeasures/data/models/planning/process.py similarity index 82% rename from flexmeasures/data/models/planning/shiftable_load.py rename to flexmeasures/data/models/planning/process.py index 0a1350cd8..ce8b892b1 100644 --- a/flexmeasures/data/models/planning/shiftable_load.py +++ b/flexmeasures/data/models/planning/process.py @@ -10,30 +10,30 @@ from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.schemas.scheduling.shiftable_load import ( - ShiftableLoadFlexModelSchema, - LoadType, +from flexmeasures.data.schemas.scheduling.process import ( + ProcessSchedulerFlexModelSchema, + ProcessType, OptimizationSense, ) from flexmeasures.data.schemas.scheduling import FlexContextSchema -class ShiftableLoadScheduler(Scheduler): +class ProcessScheduler(Scheduler): __version__ = "1" __author__ = "Seita" def compute(self) -> pd.Series | None: - """Schedule a fix load, defined as a `power` and a `duration`, within the specified time window. + """Schedule a prrocess, defined as a `power` and a `duration`, within the specified time window. To schedule a battery, please, refer to the StorageScheduler. For example, this scheduler can plan the start of a process of type `Shiftable` that lasts 5h and requires a power of 10kW. In that case, the scheduler will find the best (as to minimize/maximize the cost) hour to start the process. - This scheduler supports three types of `load_types`: - - Inflexible: this load requires to be scheduled as soon as possible. - - Breakable: this load can be divisible in smaller consumption periods. - - Shiftable: this load can start at any time within the specified time window. + This scheduler supports three types of `process_types`: + - Inflexible: this process needs to be scheduled as soon as possible. + - Breakable: this process can be divisible in smaller consumption periods. + - Shiftable: this process can start at any time within the specified time window. The resulting schedule provides the power flow at each time period. @@ -42,12 +42,12 @@ def compute(self) -> pd.Series | None: consumption_price_sensor: it defines the utility (economic, environmental, ) in each time period. It has units of quantity/energy, for example, EUR/kWh. - power: nominal power of the load. - duration: time that the load last. + power: nominal power of the process. + duration: time that the process last. optimization_sense: objective of the scheduler, to maximize or minimize. - time_restrictions: time periods in which the load cannot be schedule to. - load_type: Inflexible, Breakable or Shiftable. + time_restrictions: time periods in which the process cannot be schedule to. + process_type: Inflexible, Breakable or Shiftable. :returns: The computed schedule. """ @@ -67,7 +67,7 @@ def compute(self) -> pd.Series | None: duration: timedelta = self.flex_model.get("duration") power = self.flex_model.get("power") optimization_sense = self.flex_model.get("optimization_sense") - load_type: LoadType = self.flex_model.get("load_type") + process_type: ProcessType = self.flex_model.get("process_type") time_restrictions = self.flex_model.get("time_restrictions") # get cost data @@ -107,24 +107,24 @@ def compute(self) -> pd.Series | None: schedule[:] = energy return schedule - if load_type in [LoadType.INFLEXIBLE, LoadType.SHIFTABLE]: + if process_type in [ProcessType.INFLEXIBLE, ProcessType.SHIFTABLE]: start_time_restrictions = ( self.block_invalid_starting_times_for_whole_process_scheduling( - load_type, time_restrictions, duration, rows_to_fill + process_type, time_restrictions, duration, rows_to_fill ) ) - else: # LoadType.BREAKABLE + else: # ProcessType.BREAKABLE if (~time_restrictions).sum() < rows_to_fill: raise ValueError( "Cannot allocate a block of time {duration} given the time restrictions provided." ) # create schedule - if load_type == LoadType.INFLEXIBLE: + if process_type == ProcessType.INFLEXIBLE: self.compute_inflexible( schedule, start_time_restrictions, rows_to_fill, energy ) - elif load_type == LoadType.BREAKABLE: + elif process_type == ProcessType.BREAKABLE: self.compute_breakable( schedule, optimization_sense, @@ -133,7 +133,7 @@ def compute(self) -> pd.Series | None: rows_to_fill, energy, ) - elif load_type == LoadType.SHIFTABLE: + elif process_type == ProcessType.SHIFTABLE: self.compute_shiftable( schedule, optimization_sense, @@ -143,23 +143,23 @@ def compute(self) -> pd.Series | None: energy, ) else: - raise ValueError(f"Unknown load type '{load_type}'") + raise ValueError(f"Unknown process type '{process_type}'") return schedule.tz_convert(self.start.tzinfo) def block_invalid_starting_times_for_whole_process_scheduling( self, - load_type: LoadType, + process_type: ProcessType, time_restrictions: pd.Series, duration: timedelta, rows_to_fill: int, ) -> pd.Series: - """Blocks time periods where the load cannot be schedule into, making - sure no other time restrictions runs in the middle of the activation of the load + """Blocks time periods where the process cannot be schedule into, making + sure no other time restrictions runs in the middle of the activation of the process More technically, this function applying an erosion of the time_restrictions array with a block of length duration. - Then, the condition if time_restrictions.sum() == len(time_restrictions):, makes sure that at least we have a spot to place the load. + Then, the condition if time_restrictions.sum() == len(time_restrictions):, makes sure that at least we have a spot to place the process. For example: @@ -171,10 +171,10 @@ def block_invalid_starting_times_for_whole_process_scheduling( We can only fit a block of duration = 2 in the positions 1 and 6. sum(start_time_restrictions) == 8, while the len(time_restriction) == 10, which means we have 10-8=2 positions. - :param load_type: INFLEXIBLE, SHIFTABLE or BREAKABLE - :param time_restrictions: boolean time series indicating time periods in which the load cannot be scheduled. + :param process_type: INFLEXIBLE, SHIFTABLE or BREAKABLE + :param time_restrictions: boolean time series indicating time periods in which the process cannot be scheduled. :param duration: (datetime) duration of the length - :param rows_to_fill: (int) time periods that the load lasts + :param rows_to_fill: (int) time periods that the process lasts :return: filtered time restrictions """ @@ -201,7 +201,7 @@ def compute_inflexible( rows_to_fill: int, energy: float, ) -> None: - """Schedule load as early as possible.""" + """Schedule process as early as possible.""" start = time_restrictions[~time_restrictions].index[0] schedule.loc[start : start + self.resolution * (rows_to_fill - 1)] = energy @@ -253,13 +253,13 @@ def compute_shiftable( schedule.loc[start : start + self.resolution * (rows_to_fill - 1)] = energy def deserialize_flex_config(self): - """Deserialize flex_model using the schema ShiftableLoadFlexModelSchema and + """Deserialize flex_model using the schema ProcessSchedulerFlexModelSchema and flex_context using FlexContextSchema """ if self.flex_model is None: self.flex_model = {} - self.flex_model = ShiftableLoadFlexModelSchema( + self.flex_model = ProcessSchedulerFlexModelSchema( start=self.start, end=self.end, sensor=self.sensor ).load(self.flex_model) diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index 85d33c229..2d7981baf 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -188,19 +188,19 @@ def add_inflexible_device_forecasts( @pytest.fixture(scope="module") -def shiftable_load(db, building, setup_sources) -> dict[str, Sensor]: +def process(db, building, setup_sources) -> dict[str, Sensor]: """ - Set up a shiftable load sensor where the output of the optimization is stored. + Set up a process sensor where the output of the optimization is stored. """ - _shiftable_load = Sensor( - name="Shiftable Load", + _process = Sensor( + name="Process", generic_asset=building, event_resolution=timedelta(hours=1), unit="kWh", ) - db.session.add(_shiftable_load) + db.session.add(_process) - return _shiftable_load + return _process def add_as_beliefs(db, sensor, values, time_slots, source): diff --git a/flexmeasures/data/models/planning/tests/test_shiftable_load.py b/flexmeasures/data/models/planning/tests/test_process.py similarity index 80% rename from flexmeasures/data/models/planning/tests/test_shiftable_load.py rename to flexmeasures/data/models/planning/tests/test_process.py index 156a25c3d..3dea9021c 100644 --- a/flexmeasures/data/models/planning/tests/test_shiftable_load.py +++ b/flexmeasures/data/models/planning/tests/test_process.py @@ -3,7 +3,7 @@ import pytz from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.models.planning.shiftable_load import ShiftableLoadScheduler +from flexmeasures.data.models.planning.process import ProcessScheduler tz = pytz.timezone("Europe/Amsterdam") @@ -13,14 +13,12 @@ @pytest.mark.parametrize( - "load_type, optimal_start", + "process_type, optimal_start", [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], ) -def test_shiftable_scheduler( - add_battery_assets, shiftable_load, load_type, optimal_start -): +def test_shiftable_scheduler(add_battery_assets, process, process_type, optimal_start): """ - Test scheduling a load of 4kW of power that last 4h using the ShiftableLoadScheduler + Test scheduling a process of 4kW of power that last 4h using the ProcessScheduler without time restrictions. """ @@ -29,7 +27,7 @@ def test_shiftable_scheduler( flex_model = { "duration": "PT4H", - "load-type": load_type, + "process-type": process_type, "power": 4, } @@ -37,8 +35,8 @@ def test_shiftable_scheduler( "consumption-price-sensor": epex_da.id, } - scheduler = ShiftableLoadScheduler( - shiftable_load, + scheduler = ProcessScheduler( + process, start, end, resolution, @@ -58,14 +56,14 @@ def test_shiftable_scheduler( @pytest.mark.parametrize( - "load_type, optimal_start", + "process_type, optimal_start", [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], ) def test_duration_exceeds_planning_window( - add_battery_assets, shiftable_load, load_type, optimal_start + add_battery_assets, process, process_type, optimal_start ): """ - Test scheduling a load that last longer than the planning window. + Test scheduling a process that last longer than the planning window. """ # get the sensors from the database @@ -73,7 +71,7 @@ def test_duration_exceeds_planning_window( flex_model = { "duration": "PT48H", - "load-type": load_type, + "process-type": process_type, "power": 4, } @@ -81,8 +79,8 @@ def test_duration_exceeds_planning_window( "consumption-price-sensor": epex_da.id, } - scheduler = ShiftableLoadScheduler( - shiftable_load, + scheduler = ProcessScheduler( + process, start, end, resolution, @@ -96,9 +94,9 @@ def test_duration_exceeds_planning_window( assert (schedule == 4).all() -def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_load): +def test_shiftable_scheduler_time_restrictions(add_battery_assets, process): """ - Test ShiftableLoadScheduler with a time restrictions consisting of a block of 2h starting + Test ProcessScheduler with a time restrictions consisting of a block of 2h starting at 8am. The resulting schedules avoid the 8am-10am period and schedules for a valid period. """ @@ -109,7 +107,7 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa flex_model = { "duration": "PT4H", - "load-type": "SHIFTABLE", + "process-type": "SHIFTABLE", "power": 4, "time-restrictions": [ {"start": "2015-01-02T08:00:00+01:00", "duration": "PT2H"} @@ -119,8 +117,8 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa "consumption-price-sensor": epex_da.id, } - scheduler = ShiftableLoadScheduler( - shiftable_load, + scheduler = ProcessScheduler( + process, start, end, resolution, @@ -145,9 +143,9 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, shiftable_loa assert (schedule[time_restrictions] == 0).all() -def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_load): +def test_breakable_scheduler_time_restrictions(add_battery_assets, process): """ - Test breakable load_type of ShiftableLoadScheduler by introducing four 1-hour restrictions + Test breakable process_type of ProcessScheduler by introducing four 1-hour restrictions interspaced by 1 hour. The equivalent mask would be the following: [0,...,0,1,0,1,0,1,0,1,0, ...,0]. Trying to get the best prices (between 9am and 4pm), his makes the schedule choose time periods between the time restrictions. @@ -160,7 +158,7 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa flex_model = { "duration": "PT4H", - "load-type": "BREAKABLE", + "process-type": "BREAKABLE", "power": 4, "time-restrictions": [ {"start": "2015-01-02T09:00:00+01:00", "duration": "PT1H"}, @@ -174,8 +172,8 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa "consumption-price-sensor": epex_da.id, } - scheduler = ShiftableLoadScheduler( - shiftable_load, + scheduler = ProcessScheduler( + process, start, end, resolution, @@ -196,7 +194,7 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa @pytest.mark.parametrize( - "load_type, time_restrictions", + "process_type, time_restrictions", [ ("BREAKABLE", [{"start": "2015-01-02T00:00:00+01:00", "duration": "PT24H"}]), ("INFLEXIBLE", [{"start": "2015-01-02T03:00:00+01:00", "duration": "PT21H"}]), @@ -204,7 +202,7 @@ def test_breakable_scheduler_time_restrictions(add_battery_assets, shiftable_loa ], ) def test_impossible_schedules( - add_battery_assets, shiftable_load, load_type, time_restrictions + add_battery_assets, process, process_type, time_restrictions ): """ Test schedules with time restrictions that make a 4h block not fit anytime during the @@ -216,7 +214,7 @@ def test_impossible_schedules( flex_model = { "duration": "PT4H", - "load-type": load_type, + "process-type": process_type, "power": 4, "time-restrictions": time_restrictions, } @@ -224,8 +222,8 @@ def test_impossible_schedules( "consumption-price-sensor": epex_da.id, } - scheduler = ShiftableLoadScheduler( - shiftable_load, + scheduler = ProcessScheduler( + process, start, end, resolution, diff --git a/flexmeasures/data/schemas/scheduling/shiftable_load.py b/flexmeasures/data/schemas/scheduling/process.py similarity index 75% rename from flexmeasures/data/schemas/scheduling/shiftable_load.py rename to flexmeasures/data/schemas/scheduling/process.py index 85675fd7a..6c8517803 100644 --- a/flexmeasures/data/schemas/scheduling/shiftable_load.py +++ b/flexmeasures/data/schemas/scheduling/process.py @@ -21,7 +21,7 @@ from enum import Enum -class LoadType(Enum): +class ProcessType(Enum): INFLEXIBLE = "INFLEXIBLE" BREAKABLE = "BREAKABLE" SHIFTABLE = "SHIFTABLE" @@ -32,14 +32,14 @@ class OptimizationSense(Enum): MIN = "MIN" -class ShiftableLoadFlexModelSchema(Schema): - # time that the load last. +class ProcessSchedulerFlexModelSchema(Schema): + # time that the process last. duration = DurationField(required=True) - # nominal power of the load. + # nominal power of the process. power = fields.Float(required=True) - # policy to schedule a load: INFLEXIBLE, SHIFTABLE, BREAKABLE - load_type = fields.Enum( - LoadType, load_default=LoadType.INFLEXIBLE, data_key="load-type" + # policy to schedule a process: INFLEXIBLE, SHIFTABLE, BREAKABLE + process_type = fields.Enum( + ProcessType, load_default=ProcessType.INFLEXIBLE, data_key="process-type" ) # time_restrictions will be turned into a Series with Boolean values (where True means restricted for scheduling). time_restrictions = fields.List( @@ -56,7 +56,7 @@ class ShiftableLoadFlexModelSchema(Schema): def __init__(self, sensor: Sensor, start: datetime, end: datetime, *args, **kwargs): """Pass start and end to convert time_restrictions into a time series and sensor - as a fallback mechanism for the load_type + as a fallback mechanism for the process_type """ self.start = start.astimezone(pytz.utc) self.end = end.astimezone(pytz.utc) @@ -101,20 +101,20 @@ def post_load_time_restrictions(self, data: dict, **kwargs) -> dict: return data @pre_load - def pre_load_load_type(self, data: dict, **kwargs) -> dict: - """Fallback mechanism for the load_type variable. If not found in data, + def pre_load_process_type(self, data: dict, **kwargs) -> dict: + """Fallback mechanism for the process_type variable. If not found in data, it tries to find it in among the sensor or asset attributes and, if it's not found there either, it defaults to "INFLEXIBLE". """ - if "load-type" not in data or data["load-type"] is None: - load_type = self.sensor.get_attribute("load_type") + if "process-type" not in data or data["process-type"] is None: + process_type = self.sensor.get_attribute("process-type") - if load_type is None: - load_type = self.sensor.generic_asset.get_attribute("load_type") + if process_type is None: + process_type = self.sensor.generic_asset.get_attribute("process-type") - if load_type is None: - load_type = "INFLEXIBLE" + if process_type is None: + process_type = "INFLEXIBLE" - data["load-type"] = load_type + data["process-type"] = process_type return data diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 508c8f071..ec24eb3a0 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -1,23 +1,23 @@ -from flexmeasures.data.schemas.scheduling.shiftable_load import ( - ShiftableLoadFlexModelSchema, - LoadType, +from flexmeasures.data.schemas.scheduling.process import ( + ProcessSchedulerFlexModelSchema, + ProcessType, ) from datetime import datetime import pytz -def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): +def test_process_scheduler_flex_model_load(db, app, setup_dummy_sensors): sensor1, _ = setup_dummy_sensors - schema = ShiftableLoadFlexModelSchema( + schema = ProcessSchedulerFlexModelSchema( sensor=sensor1, start=datetime(2023, 1, 1, tzinfo=pytz.UTC), end=datetime(2023, 1, 2, tzinfo=pytz.UTC), ) - shiftable_load_flex_model = schema.load( + process_scheduler_flex_model = schema.load( { "duration": "PT4H", "power": 30.0, @@ -27,22 +27,22 @@ def test_shiftable_load_flex_model_load(db, app, setup_dummy_sensors): } ) - print(shiftable_load_flex_model) + assert process_scheduler_flex_model["process_type"] == ProcessType.INFLEXIBLE -def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): +def test_process_scheduler_flex_model_process_type(db, app, setup_dummy_sensors): sensor1, _ = setup_dummy_sensors # checking default - schema = ShiftableLoadFlexModelSchema( + schema = ProcessSchedulerFlexModelSchema( sensor=sensor1, start=datetime(2023, 1, 1, tzinfo=pytz.UTC), end=datetime(2023, 1, 2, tzinfo=pytz.UTC), ) - shiftable_load_flex_model = schema.load( + process_scheduler_flex_model = schema.load( { "duration": "PT4H", "power": 30.0, @@ -52,17 +52,17 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): } ) - assert shiftable_load_flex_model["load_type"] == LoadType.INFLEXIBLE + assert process_scheduler_flex_model["process_type"] == ProcessType.INFLEXIBLE - sensor1.attributes["load_type"] = "SHIFTABLE" + sensor1.attributes["process-type"] = "SHIFTABLE" - schema = ShiftableLoadFlexModelSchema( + schema = ProcessSchedulerFlexModelSchema( sensor=sensor1, start=datetime(2023, 1, 1, tzinfo=pytz.UTC), end=datetime(2023, 1, 2, tzinfo=pytz.UTC), ) - shiftable_load_flex_model = schema.load( + process_scheduler_flex_model = schema.load( { "duration": "PT4H", "power": 30.0, @@ -72,4 +72,4 @@ def test_shiftable_load_flex_model_load_type(db, app, setup_dummy_sensors): } ) - assert shiftable_load_flex_model["load_type"] == LoadType.SHIFTABLE + assert process_scheduler_flex_model["process_type"] == ProcessType.SHIFTABLE diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 6b077e9bd..df56e3c9a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -19,7 +19,7 @@ from flexmeasures.data import db from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.models.planning.storage import StorageScheduler -from flexmeasures.data.models.planning.shiftable_load import ShiftableLoadScheduler +from flexmeasures.data.models.planning.process import ProcessScheduler from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.utils import get_data_source, save_to_db @@ -195,8 +195,8 @@ def find_scheduler_class(sensor: Sensor) -> type: "two-way_evse", ): scheduler_class = StorageScheduler - elif sensor.generic_asset.generic_asset_type.name in ("LoadType"): - scheduler_class = ShiftableLoadScheduler + elif sensor.generic_asset.generic_asset_type.name in ("process", "load"): + scheduler_class = ProcessScheduler else: raise ValueError( "Scheduling is not (yet) supported for asset type %s." From 6aeb7efb9024175a9ab4cb2420bc38472d7f2552 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 26 Jul 2023 23:19:11 +0200 Subject: [PATCH 31/48] rename shiftable_load to process Signed-off-by: Victor Garcia Reolid --- documentation/cli/change_log.rst | 2 +- documentation/cli/commands.rst | 2 +- flexmeasures/cli/data_add.py | 40 +++++++++---------- flexmeasures/cli/tests/conftest.py | 16 ++++---- flexmeasures/cli/tests/test_data_add.py | 28 ++++++------- .../models/planning/tests/test_process.py | 4 +- 6 files changed, 45 insertions(+), 47 deletions(-) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 05304c6eb..92f2175bc 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -8,7 +8,7 @@ since v0.15.0 | July XX, 2023 ================================= * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times. -* Add ``flexmeasures add schedule for-shiftable`` to create a new shiftable load schedule for a given power sensor. +* Add ``flexmeasures add schedule for-process`` to create a new process schedule for a given power sensor. since v0.14.1 | June XX, 2023 diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 6ac8dcdc8..a050fb3e8 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -36,7 +36,7 @@ of which some are referred to in this documentation. ``flexmeasures add source`` Add a new data source. ``flexmeasures add forecasts`` Create forecasts. ``flexmeasures add schedule for-storage`` Create a charging schedule for a storage asset. -``flexmeasures add schedule for-shiftable`` Create a schedule for a shiftable load. +``flexmeasures add schedule for-process`` Create a schedule for a process asset. ``flexmeasures add holidays`` Add holiday annotations to accounts and/or assets. ``flexmeasures add annotation`` Add annotation to accounts, assets and/or sensors. ``flexmeasures add toy-account`` Create a toy account, for tutorials and trying things. diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 74dbdda23..98d47c99e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1160,7 +1160,7 @@ def add_schedule_for_storage( click.secho("New schedule is stored.", **MsgStyle.SUCCESS) -@create_schedule.command("for-shiftable") +@create_schedule.command("for-process") @with_appcontext @click.option( "--sensor-id", @@ -1191,26 +1191,26 @@ def add_schedule_for_storage( help="Duration of schedule, after --start. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", ) @click.option( - "--load-duration", - "load_duration", + "--process-duration", + "process_duration", type=DurationField(), required=True, - help="Duration of the load. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", + help="Duration of the process. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", ) @click.option( - "--load-type", - "load_type", + "--process-type", + "process_type", type=click.Choice(["INFLEXIBLE", "BREAKABLE", "SHIFTABLE"], case_sensitive=False), required=False, default="SHIFTABLE", - help="Load shift policy type: INFLEXIBLE, BREAKABLE or SHIFTABLE.", + help="Process schedule policy: INFLEXIBLE, BREAKABLE or SHIFTABLE.", ) @click.option( - "--load-power", - "load_power", + "--process-power", + "process_power", type=ur.Quantity, required=True, - help="Constant power of the load during the activation period, e.g. 4kW.", + help="Constant power of the process during the activation period, e.g. 4kW.", ) @click.option( "--forbid", @@ -1227,22 +1227,22 @@ def add_schedule_for_storage( help="Whether to queue a scheduling job instead of computing directly. " "To process the job, run a worker (on any computer, but configured to the same databases) to process the 'scheduling' queue. Defaults to False.", ) -def add_schedule_shiftable_load( +def add_schedule_process( power_sensor: Sensor, consumption_price_sensor: Sensor, start: datetime, duration: timedelta, - load_duration: timedelta, - load_type: str, - load_power: ur.Quantity, + process_duration: timedelta, + process_type: str, + process_power: ur.Quantity, forbid: List | None = None, as_job: bool = False, ): - """Create a new schedule for a shiftable asset. + """Create a new schedule for a process asset. Current limitations: - Only supports consumption blocks. - - Not taking into account grid constraints or other loads. + - Not taking into account grid constraints or other processes. """ if forbid is None: @@ -1258,7 +1258,7 @@ def add_schedule_shiftable_load( end = start + duration - load_power = convert_units(load_power.magnitude, load_power.units, "MW") # type: ignore + process_power = convert_units(process_power.magnitude, process_power.units, "MW") # type: ignore scheduling_kwargs = dict( start=start, @@ -1266,9 +1266,9 @@ def add_schedule_shiftable_load( belief_time=server_now(), resolution=power_sensor.event_resolution, flex_model={ - "duration": pd.Timedelta(load_duration).isoformat(), - "load-type": load_type, - "power": load_power, + "duration": pd.Timedelta(process_duration).isoformat(), + "process-type": process_type, + "power": process_power, "time-restrictions": [TimeIntervalSchema().dump(f) for f in forbid], }, flex_context={ diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 1ac161344..83eea3d18 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -100,26 +100,26 @@ def reporter_config_raw(app, db, setup_dummy_data): @pytest.mark.skip_github @pytest.fixture(scope="module") -def shiftable_load_power_sensor(db, app): +def process_power_sensor(db, app): """ - Create an asset of type "LoadType" and a power sensor to hold the result of + Create an asset of type "ProcessType" and a power sensor to hold the result of the scheduler. """ - shiftable_load_asset_type = GenericAssetType(name="LoadType") + process_asset_type = GenericAssetType(name="process") - db.session.add(shiftable_load_asset_type) + db.session.add(process_asset_type) - shiftable_asset = GenericAsset( - name="Test Asset", generic_asset_type=shiftable_load_asset_type + processasset = GenericAsset( + name="Test Asset", generic_asset_type=process_asset_type ) - db.session.add(shiftable_asset) + db.session.add(processasset) power_sensor = Sensor( "power", - generic_asset=shiftable_asset, + generic_asset=processasset, event_resolution=timedelta(hours=1), unit="MW", ) diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 9b14b0564..3e293854b 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -214,28 +214,26 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): @pytest.mark.skip_github -@pytest.mark.parametrize("load_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) -def test_add_shiftable( - app, db, shiftable_load_power_sensor, add_market_prices, load_type -): +@pytest.mark.parametrize("process_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) +def test_add_process(app, db, process_power_sensor, add_market_prices, process_type): """ Schedule a 4h of consumption block at a constant power of 400kW in a day using - the three shiftable policies: inflexible, shiftable and breakable. + the three process policies: inflexible, shiftable and breakable. """ - from flexmeasures.cli.data_add import add_schedule_shiftable_load + from flexmeasures.cli.data_add import add_schedule_process epex_da = Sensor.query.filter(Sensor.name == "epex_da").one_or_none() - shiftable_load_power_sensor_id = shiftable_load_power_sensor + process_power_sensor_id = process_power_sensor cli_input_params = { - "sensor-id": shiftable_load_power_sensor_id, + "sensor-id": process_power_sensor_id, "start": "2015-01-02T00:00:00+01:00", "duration": "PT24H", - "load-duration": "PT4H", - "load-power": "0.4MW", - "load-type": load_type, + "process-duration": "PT4H", + "process-power": "0.4MW", + "process-type": process_type, "consumption-price-sensor": epex_da.id, "forbid": '{"start" : "2015-01-02T00:00:00+01:00", "duration" : "PT2H"}', } @@ -244,15 +242,15 @@ def test_add_shiftable( runner = app.test_cli_runner() # call command - result = runner.invoke(add_schedule_shiftable_load, cli_input) + result = runner.invoke(add_schedule_process, cli_input) print(result) assert result.exit_code == 0 # run command without errors - shiftable_load_power_sensor = Sensor.query.get(shiftable_load_power_sensor_id) + process_power_sensor = Sensor.query.get(process_power_sensor_id) - schedule = shiftable_load_power_sensor.search_beliefs() + schedule = process_power_sensor.search_beliefs() # check if the schedule is not empty more detailed testing can be found - # in data/models/planning/tests/test_shiftable_loads.py. + # in data/models/planning/tests/test_processs.py. assert (schedule == -0.4).event_value.sum() == 4 diff --git a/flexmeasures/data/models/planning/tests/test_process.py b/flexmeasures/data/models/planning/tests/test_process.py index 3dea9021c..08a198963 100644 --- a/flexmeasures/data/models/planning/tests/test_process.py +++ b/flexmeasures/data/models/planning/tests/test_process.py @@ -16,7 +16,7 @@ "process_type, optimal_start", [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], ) -def test_shiftable_scheduler(add_battery_assets, process, process_type, optimal_start): +def test_processscheduler(add_battery_assets, process, process_type, optimal_start): """ Test scheduling a process of 4kW of power that last 4h using the ProcessScheduler without time restrictions. @@ -94,7 +94,7 @@ def test_duration_exceeds_planning_window( assert (schedule == 4).all() -def test_shiftable_scheduler_time_restrictions(add_battery_assets, process): +def test_processscheduler_time_restrictions(add_battery_assets, process): """ Test ProcessScheduler with a time restrictions consisting of a block of 2h starting at 8am. The resulting schedules avoid the 8am-10am period and schedules for a valid period. From 827df19f1c4831fb6500c35eca5b0bce90c08909 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 22:01:34 +0200 Subject: [PATCH 32/48] rename optimization_sense to optimization_direction Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/process.py | 18 +++++++++--------- .../data/schemas/scheduling/process.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flexmeasures/data/models/planning/process.py b/flexmeasures/data/models/planning/process.py index ce8b892b1..b6c385a8f 100644 --- a/flexmeasures/data/models/planning/process.py +++ b/flexmeasures/data/models/planning/process.py @@ -45,7 +45,7 @@ def compute(self) -> pd.Series | None: power: nominal power of the process. duration: time that the process last. - optimization_sense: objective of the scheduler, to maximize or minimize. + optimization_direction: objective of the scheduler, to maximize or minimize. time_restrictions: time periods in which the process cannot be schedule to. process_type: Inflexible, Breakable or Shiftable. @@ -66,7 +66,7 @@ def compute(self) -> pd.Series | None: ) duration: timedelta = self.flex_model.get("duration") power = self.flex_model.get("power") - optimization_sense = self.flex_model.get("optimization_sense") + optimization_direction = self.flex_model.get("optimization_direction") process_type: ProcessType = self.flex_model.get("process_type") time_restrictions = self.flex_model.get("time_restrictions") @@ -127,7 +127,7 @@ def compute(self) -> pd.Series | None: elif process_type == ProcessType.BREAKABLE: self.compute_breakable( schedule, - optimization_sense, + optimization_direction, time_restrictions, cost, rows_to_fill, @@ -136,7 +136,7 @@ def compute(self) -> pd.Series | None: elif process_type == ProcessType.SHIFTABLE: self.compute_shiftable( schedule, - optimization_sense, + optimization_direction, start_time_restrictions, cost, rows_to_fill, @@ -209,16 +209,16 @@ def compute_inflexible( def compute_breakable( self, schedule: pd.Series, - optimization_sense: OptimizationSense, + optimization_direction: OptimizationSense, time_restrictions: pd.Series, cost: pd.DataFrame, rows_to_fill: int, energy: float, ) -> None: - """Break up schedule and divide it over the time slots with the largest utility (max/min cost depending on optimization_sense).""" + """Break up schedule and divide it over the time slots with the largest utility (max/min cost depending on optimization_direction).""" cost = cost[~time_restrictions].reset_index() - if optimization_sense == OptimizationSense.MIN: + if optimization_direction == OptimizationSense.MIN: cost_ranking = cost.sort_values( by=["event_value", "event_start"], ascending=[True, True] ) @@ -232,7 +232,7 @@ def compute_breakable( def compute_shiftable( self, schedule: pd.Series, - optimization_sense: OptimizationSense, + optimization_direction: OptimizationSense, time_restrictions: pd.Series, cost: pd.DataFrame, rows_to_fill: int, @@ -243,7 +243,7 @@ def compute_shiftable( cost.rolling(rows_to_fill).sum().shift(-rows_to_fill + 1) ) - if optimization_sense == OptimizationSense.MIN: + if optimization_direction == OptimizationSense.MIN: start = block_cost[~time_restrictions].idxmin() else: start = block_cost[~time_restrictions].idxmax() diff --git a/flexmeasures/data/schemas/scheduling/process.py b/flexmeasures/data/schemas/scheduling/process.py index 6c8517803..43aeb4599 100644 --- a/flexmeasures/data/schemas/scheduling/process.py +++ b/flexmeasures/data/schemas/scheduling/process.py @@ -48,7 +48,7 @@ class ProcessSchedulerFlexModelSchema(Schema): load_default=[], ) # objective of the scheduler, to maximize or minimize. - optimization_sense = fields.Enum( + optimization_direction = fields.Enum( OptimizationSense, load_default=OptimizationSense.MIN, data_key="optimization-sense", From 9d7e6e81a2de809c8d843e5c021879fa90d49084 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 22:13:42 +0200 Subject: [PATCH 33/48] consistent capitalization of INFLEXIBLE, SHIFTABLE AND BREAKABLE Signed-off-by: Victor Garcia Reolid --- documentation/api/notation.rst | 6 +++--- documentation/changelog.rst | 2 +- flexmeasures/data/models/planning/process.py | 10 +++++----- .../data/models/planning/tests/test_process.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index b00580715..c910079c9 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -202,13 +202,13 @@ Here are the three types of flexibility models you can expect to be built-in: For some examples, see the `[POST] /sensors/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ endpoint docs. -2) For **shiftable processes** +2) For **processes** - ``power``: nominal power of the load. - ``duration``: time that the load last. - ``optimization_sense``: objective of the scheduler, to maximize or minimize. - ``time_restrictions``: time periods in which the load cannot be schedule to. - - ``load_type``: Inflexible, Breakable or Shiftable. + - ``process_type``: INFLEXIBLE, BREAKABLE or SHIFTABLE. 3) For **buffer devices** (e.g. thermal energy storage systems connected to heat pumps), use the same flexibility parameters described above for storage devices. Here are some tips to model a buffer with these parameters: @@ -233,7 +233,7 @@ Flex context With the flexibility context, we aim to describe the system in which the flexible assets operates: -- ``inflexible-device-sensors`` ― power sensors that are relevant, but not flexible, such as a sensor recording rooftop solar power connected behind the main meter, whose production falls under the same contract as the flexible device(s) being scheduled +- ``INFLEXIBLE-device-sensors`` ― power sensors that are relevant, but not flexible, such as a sensor recording rooftop solar power connected behind the main meter, whose production falls under the same contract as the flexible device(s) being scheduled - ``consumption-price-sensor`` ― the sensor which defines costs/revenues of consuming energy - ``production-price-sensor`` ― the sensor which defines cost/revenues of producing energy diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 81a45ed96..609cbbfc3 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] -* Add `ProcessScheduler` class, which optimizes the starting time of processes using one of the following policies: inflexible, shiftable and breakable [see `PR #729 `_] +* Add `ProcessScheduler` class, which optimizes the starting time of processes using one of the following policies: INFLEXIBLE, SHIFTABLE and BREAKABLE [see `PR #729 `_] Bugfixes ----------- diff --git a/flexmeasures/data/models/planning/process.py b/flexmeasures/data/models/planning/process.py index b6c385a8f..890debfcf 100644 --- a/flexmeasures/data/models/planning/process.py +++ b/flexmeasures/data/models/planning/process.py @@ -27,13 +27,13 @@ def compute(self) -> pd.Series | None: """Schedule a prrocess, defined as a `power` and a `duration`, within the specified time window. To schedule a battery, please, refer to the StorageScheduler. - For example, this scheduler can plan the start of a process of type `Shiftable` that lasts 5h and requires a power of 10kW. + For example, this scheduler can plan the start of a process of type `SHIFTABLE` that lasts 5h and requires a power of 10kW. In that case, the scheduler will find the best (as to minimize/maximize the cost) hour to start the process. This scheduler supports three types of `process_types`: - - Inflexible: this process needs to be scheduled as soon as possible. - - Breakable: this process can be divisible in smaller consumption periods. - - Shiftable: this process can start at any time within the specified time window. + - INFLEXIBLE: this process needs to be scheduled as soon as possible. + - BREAKABLE: this process can be divisible in smaller consumption periods. + - SHIFTABLE: this process can start at any time within the specified time window. The resulting schedule provides the power flow at each time period. @@ -47,7 +47,7 @@ def compute(self) -> pd.Series | None: optimization_direction: objective of the scheduler, to maximize or minimize. time_restrictions: time periods in which the process cannot be schedule to. - process_type: Inflexible, Breakable or Shiftable. + process_type: INFLEXIBLE, BREAKABLE or SHIFTABLE. :returns: The computed schedule. """ diff --git a/flexmeasures/data/models/planning/tests/test_process.py b/flexmeasures/data/models/planning/tests/test_process.py index 3dea9021c..be6210084 100644 --- a/flexmeasures/data/models/planning/tests/test_process.py +++ b/flexmeasures/data/models/planning/tests/test_process.py @@ -145,7 +145,7 @@ def test_shiftable_scheduler_time_restrictions(add_battery_assets, process): def test_breakable_scheduler_time_restrictions(add_battery_assets, process): """ - Test breakable process_type of ProcessScheduler by introducing four 1-hour restrictions + Test BREAKABLE process_type of ProcessScheduler by introducing four 1-hour restrictions interspaced by 1 hour. The equivalent mask would be the following: [0,...,0,1,0,1,0,1,0,1,0, ...,0]. Trying to get the best prices (between 9am and 4pm), his makes the schedule choose time periods between the time restrictions. From 8f7c276c8bd47c675a4ec8a251817ea02e2a241d Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 22:41:47 +0200 Subject: [PATCH 34/48] fix typo Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/process.py b/flexmeasures/data/models/planning/process.py index 890debfcf..7921cc4dc 100644 --- a/flexmeasures/data/models/planning/process.py +++ b/flexmeasures/data/models/planning/process.py @@ -24,7 +24,7 @@ class ProcessScheduler(Scheduler): __author__ = "Seita" def compute(self) -> pd.Series | None: - """Schedule a prrocess, defined as a `power` and a `duration`, within the specified time window. + """Schedule a process, defined as a `power` and a `duration`, within the specified time window. To schedule a battery, please, refer to the StorageScheduler. For example, this scheduler can plan the start of a process of type `SHIFTABLE` that lasts 5h and requires a power of 10kW. From 2123ac54c2a6bc9651c1e4d928ba4cd1bc0bd5d1 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 22:51:46 +0200 Subject: [PATCH 35/48] harmonize capitalization Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/test_data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 3e293854b..0a9fa79a3 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -218,7 +218,7 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): def test_add_process(app, db, process_power_sensor, add_market_prices, process_type): """ Schedule a 4h of consumption block at a constant power of 400kW in a day using - the three process policies: inflexible, shiftable and breakable. + the three process policies: INFLEXIBLE, SHIFTABLE and BREAKABLE. """ from flexmeasures.cli.data_add import add_schedule_process From e5cad355a8e8942ff829b44f46822c094b30875e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 22:52:30 +0200 Subject: [PATCH 36/48] fix capitalizationof `inflexible-device-sensors` Signed-off-by: Victor Garcia Reolid --- documentation/api/notation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index c910079c9..0e5a6c8a7 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -233,7 +233,7 @@ Flex context With the flexibility context, we aim to describe the system in which the flexible assets operates: -- ``INFLEXIBLE-device-sensors`` ― power sensors that are relevant, but not flexible, such as a sensor recording rooftop solar power connected behind the main meter, whose production falls under the same contract as the flexible device(s) being scheduled +- ``inflexible-device-sensors`` ― power sensors that are relevant, but not flexible, such as a sensor recording rooftop solar power connected behind the main meter, whose production falls under the same contract as the flexible device(s) being scheduled - ``consumption-price-sensor`` ― the sensor which defines costs/revenues of consuming energy - ``production-price-sensor`` ― the sensor which defines cost/revenues of producing energy From 2cf72a9f04f0cc80e7e64f35604c66c78fe0474c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 23:28:32 +0200 Subject: [PATCH 37/48] update changelog Signed-off-by: Victor Garcia Reolid --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 25e720fab..654b4dcb6 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,7 +17,7 @@ New features * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 `_] * Added API endpoint `/sensor/` for fetching a single sensor. [see `PR #759 `_] * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 `_] -* Add `ProcessScheduler` class and CLI command `flexmeasures add schedule for-process` to optimize the starting time of processes one of the following policies: INFLEXIBLE, SHIFTABLE and BREAKABLE [see `PR #729 `_ and `PR #768 `_] +* Add `ProcessScheduler` class to optimize the starting time of processes one of the policies developed (INFLEXIBLE, SHIFTABLE and BREAKABLE), accessible via the CLI command `flexmeasures add schedule for-process` [see `PR #729 `_ and `PR #768 `_] Bugfixes ----------- From 3769d3c3e9c5f35274b82b2894e4facc604deebf Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 23:35:19 +0200 Subject: [PATCH 38/48] fix potental bug Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/data_add.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 98d47c99e..8f1eea4e1 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1271,11 +1271,13 @@ def add_schedule_process( "power": process_power, "time-restrictions": [TimeIntervalSchema().dump(f) for f in forbid], }, - flex_context={ - "consumption-price-sensor": consumption_price_sensor.id, - }, ) + if consumption_price_sensor is not None: + scheduling_kwargs["flex_context"] = { + "consumption-price-sensor": consumption_price_sensor.id, + } + if as_job: job = create_scheduling_job(sensor=power_sensor, **scheduling_kwargs) if job: From ba189c420043bcdae27fa64034f38e311647209a Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 27 Jul 2023 23:47:52 +0200 Subject: [PATCH 39/48] add underscores Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 8 ++++---- flexmeasures/data/models/planning/tests/test_process.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 83eea3d18..fe86f2abf 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -102,7 +102,7 @@ def reporter_config_raw(app, db, setup_dummy_data): @pytest.fixture(scope="module") def process_power_sensor(db, app): """ - Create an asset of type "ProcessType" and a power sensor to hold the result of + Create an asset of type "process" and a power sensor to hold the result of the scheduler. """ @@ -111,15 +111,15 @@ def process_power_sensor(db, app): db.session.add(process_asset_type) - processasset = GenericAsset( + process_asset = GenericAsset( name="Test Asset", generic_asset_type=process_asset_type ) - db.session.add(processasset) + db.session.add(process_asset) power_sensor = Sensor( "power", - generic_asset=processasset, + generic_asset=process_asset, event_resolution=timedelta(hours=1), unit="MW", ) diff --git a/flexmeasures/data/models/planning/tests/test_process.py b/flexmeasures/data/models/planning/tests/test_process.py index ba0e8618c..aba8b3308 100644 --- a/flexmeasures/data/models/planning/tests/test_process.py +++ b/flexmeasures/data/models/planning/tests/test_process.py @@ -16,7 +16,7 @@ "process_type, optimal_start", [("INFLEXIBLE", datetime(2015, 1, 2, 0)), ("SHIFTABLE", datetime(2015, 1, 2, 8))], ) -def test_processscheduler(add_battery_assets, process, process_type, optimal_start): +def test_process_scheduler(add_battery_assets, process, process_type, optimal_start): """ Test scheduling a process of 4kW of power that last 4h using the ProcessScheduler without time restrictions. @@ -94,7 +94,7 @@ def test_duration_exceeds_planning_window( assert (schedule == 4).all() -def test_processscheduler_time_restrictions(add_battery_assets, process): +def test_process_scheduler_time_restrictions(add_battery_assets, process): """ Test ProcessScheduler with a time restrictions consisting of a block of 2h starting at 8am. The resulting schedules avoid the 8am-10am period and schedules for a valid period. From db85ce404d2dd106f2fd78e7503a1a75a11aa5ee Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 28 Jul 2023 09:31:55 +0200 Subject: [PATCH 40/48] fix test Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 29 +++++++++++++++++++++---- flexmeasures/cli/tests/test_data_add.py | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index fe86f2abf..300b6541c 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -2,11 +2,14 @@ from datetime import datetime, timedelta from pytz import utc +import pandas as pd from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flexmeasures.data.models.planning.utils import initialize_index + @pytest.fixture(scope="module") @pytest.mark.skip_github @@ -100,10 +103,10 @@ def reporter_config_raw(app, db, setup_dummy_data): @pytest.mark.skip_github @pytest.fixture(scope="module") -def process_power_sensor(db, app): +def process_power_sensor(db, app, setup_markets, setup_sources): """ - Create an asset of type "process" and a power sensor to hold the result of - the scheduler. + Create an asset of type "process", power sensor to hold the result of + the scheduler and price data consisting of 8 expensive hours, 8 cheap hours, and again 8 expensive hours- """ @@ -112,7 +115,7 @@ def process_power_sensor(db, app): db.session.add(process_asset_type) process_asset = GenericAsset( - name="Test Asset", generic_asset_type=process_asset_type + name="Test Process Asset", generic_asset_type=process_asset_type ) db.session.add(process_asset) @@ -126,6 +129,24 @@ def process_power_sensor(db, app): db.session.add(power_sensor) + time_slots = initialize_index( + start=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"), + end=pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam"), + resolution="1H", + ) + values = [100] * 8 + [90] * 8 + [100] * 8 + beliefs = [ + TimedBelief( + event_start=dt, + belief_horizon=timedelta(hours=0), + event_value=val, + source=setup_sources["Seita"], + sensor=setup_markets["epex_da"].corresponding_sensor, + ) + for dt, val in zip(time_slots, values) + ] + db.session.add_all(beliefs) + db.session.commit() yield power_sensor.id diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 0a9fa79a3..3cf35c04c 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -215,7 +215,7 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): @pytest.mark.skip_github @pytest.mark.parametrize("process_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) -def test_add_process(app, db, process_power_sensor, add_market_prices, process_type): +def test_add_process(app, db, process_power_sensor, process_type): """ Schedule a 4h of consumption block at a constant power of 400kW in a day using the three process policies: INFLEXIBLE, SHIFTABLE and BREAKABLE. From 92b5d9766f2c6eaaa0d3e383178637be7ebcc909 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Mon, 31 Jul 2023 09:41:58 +0200 Subject: [PATCH 41/48] rename OptimizationSense to OptimizationDirection Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/schemas/scheduling/process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/process.py b/flexmeasures/data/schemas/scheduling/process.py index 43aeb4599..48906b135 100644 --- a/flexmeasures/data/schemas/scheduling/process.py +++ b/flexmeasures/data/schemas/scheduling/process.py @@ -27,7 +27,7 @@ class ProcessType(Enum): SHIFTABLE = "SHIFTABLE" -class OptimizationSense(Enum): +class OptimizationDirection(Enum): MAX = "MAX" MIN = "MIN" @@ -49,8 +49,8 @@ class ProcessSchedulerFlexModelSchema(Schema): ) # objective of the scheduler, to maximize or minimize. optimization_direction = fields.Enum( - OptimizationSense, - load_default=OptimizationSense.MIN, + OptimizationDirection, + load_default=OptimizationDirection.MIN, data_key="optimization-sense", ) From d11a191ddbb864ba624265f0bf6f14fcda4af0b8 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Mon, 31 Jul 2023 10:27:54 +0200 Subject: [PATCH 42/48] fix missing optimization direction renaming Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/process.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/process.py b/flexmeasures/data/models/planning/process.py index 7921cc4dc..ac92f37bc 100644 --- a/flexmeasures/data/models/planning/process.py +++ b/flexmeasures/data/models/planning/process.py @@ -13,7 +13,7 @@ from flexmeasures.data.schemas.scheduling.process import ( ProcessSchedulerFlexModelSchema, ProcessType, - OptimizationSense, + OptimizationDirection, ) from flexmeasures.data.schemas.scheduling import FlexContextSchema @@ -103,7 +103,13 @@ def compute(self) -> pd.Series | None: ceil(duration / consumption_price_sensor.event_resolution), len(schedule) ) + # duration of the process exceeds the scheduling window if rows_to_fill == len(schedule): + if time_restrictions.sum() > 0: + raise ValueError( + "Cannot handle time restrictions if the duration of the process exceeds that of the schedule window." + ) + schedule[:] = energy return schedule @@ -209,7 +215,7 @@ def compute_inflexible( def compute_breakable( self, schedule: pd.Series, - optimization_direction: OptimizationSense, + optimization_direction: OptimizationDirection, time_restrictions: pd.Series, cost: pd.DataFrame, rows_to_fill: int, @@ -218,7 +224,7 @@ def compute_breakable( """Break up schedule and divide it over the time slots with the largest utility (max/min cost depending on optimization_direction).""" cost = cost[~time_restrictions].reset_index() - if optimization_direction == OptimizationSense.MIN: + if optimization_direction == OptimizationDirection.MIN: cost_ranking = cost.sort_values( by=["event_value", "event_start"], ascending=[True, True] ) @@ -232,7 +238,7 @@ def compute_breakable( def compute_shiftable( self, schedule: pd.Series, - optimization_direction: OptimizationSense, + optimization_direction: OptimizationDirection, time_restrictions: pd.Series, cost: pd.DataFrame, rows_to_fill: int, @@ -243,7 +249,7 @@ def compute_shiftable( cost.rolling(rows_to_fill).sum().shift(-rows_to_fill + 1) ) - if optimization_direction == OptimizationSense.MIN: + if optimization_direction == OptimizationDirection.MIN: start = block_cost[~time_restrictions].idxmin() else: start = block_cost[~time_restrictions].idxmax() From 159d33b03df0e4c19af6798ae4ce98db3febd7c7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Mon, 31 Jul 2023 13:03:53 +0200 Subject: [PATCH 43/48] simplify test Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 23 +---------------------- flexmeasures/cli/tests/test_data_add.py | 6 +++--- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 300b6541c..5b1f8b90b 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -2,14 +2,11 @@ from datetime import datetime, timedelta from pytz import utc -import pandas as pd from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.data.models.planning.utils import initialize_index - @pytest.fixture(scope="module") @pytest.mark.skip_github @@ -103,7 +100,7 @@ def reporter_config_raw(app, db, setup_dummy_data): @pytest.mark.skip_github @pytest.fixture(scope="module") -def process_power_sensor(db, app, setup_markets, setup_sources): +def process_power_sensor(db, app, add_market_prices): """ Create an asset of type "process", power sensor to hold the result of the scheduler and price data consisting of 8 expensive hours, 8 cheap hours, and again 8 expensive hours- @@ -129,24 +126,6 @@ def process_power_sensor(db, app, setup_markets, setup_sources): db.session.add(power_sensor) - time_slots = initialize_index( - start=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"), - end=pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam"), - resolution="1H", - ) - values = [100] * 8 + [90] * 8 + [100] * 8 - beliefs = [ - TimedBelief( - event_start=dt, - belief_horizon=timedelta(hours=0), - event_value=val, - source=setup_sources["Seita"], - sensor=setup_markets["epex_da"].corresponding_sensor, - ) - for dt, val in zip(time_slots, values) - ] - db.session.add_all(beliefs) - db.session.commit() yield power_sensor.id diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 3cf35c04c..77935deea 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -16,7 +16,7 @@ @pytest.mark.skip_github -def test_add_annotation(app, db, setup_roles_users): +def test_add_annotation(app, fresh_db, setup_roles_users_fresh_db): from flexmeasures.cli.data_add import add_annotation cli_input = { @@ -52,7 +52,7 @@ def test_add_annotation(app, db, setup_roles_users): @pytest.mark.skip_github -def test_add_holidays(app, db, setup_roles_users): +def test_add_holidays(app, fresh_db, setup_roles_users_fresh_db): from flexmeasures.cli.data_add import add_holidays cli_input = { @@ -215,7 +215,7 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): @pytest.mark.skip_github @pytest.mark.parametrize("process_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) -def test_add_process(app, db, process_power_sensor, process_type): +def test_add_process(app, process_power_sensor, process_type): """ Schedule a 4h of consumption block at a constant power of 400kW in a day using the three process policies: INFLEXIBLE, SHIFTABLE and BREAKABLE. From 30923899628f06fb2b9bc03ceded5720d5915122 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Aug 2023 13:54:00 +0200 Subject: [PATCH 44/48] test run pytest ci Signed-off-by: Victor Garcia Reolid --- flexmeasures/api/dev/tests/conftest.py | 7 +++++-- flexmeasures/api/v3_0/tests/conftest.py | 6 ++++-- flexmeasures/api/v3_0/tests/test_sensor_data.py | 8 ++++++-- flexmeasures/cli/tests/conftest.py | 2 +- flexmeasures/cli/tests/test_data_add.py | 14 +++++++++----- flexmeasures/conftest.py | 15 ++++++++------- flexmeasures/data/tests/conftest.py | 3 ++- 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/flexmeasures/api/dev/tests/conftest.py b/flexmeasures/api/dev/tests/conftest.py index 9b98647af..540859eb3 100644 --- a/flexmeasures/api/dev/tests/conftest.py +++ b/flexmeasures/api/dev/tests/conftest.py @@ -1,5 +1,6 @@ import pytest +from flexmeasures import User from flexmeasures.api.v3_0.tests.conftest import add_incineration_line from flexmeasures.data.models.time_series import Sensor @@ -10,7 +11,7 @@ def setup_api_test_data(db, setup_roles_users, setup_generic_assets): Set up data for API dev tests. """ print("Setting up data for API dev tests on %s" % db.engine) - add_incineration_line(db, setup_roles_users["Test Supplier User"]) + add_incineration_line(db, User.query.get(setup_roles_users["Test Supplier User"])) @pytest.fixture(scope="function") @@ -23,4 +24,6 @@ def setup_api_fresh_test_data( print("Setting up fresh data for API dev tests on %s" % fresh_db.engine) for sensor in Sensor.query.all(): fresh_db.delete(sensor) - add_incineration_line(fresh_db, setup_roles_users_fresh_db["Test Supplier User"]) + add_incineration_line( + fresh_db, User.query.get(setup_roles_users_fresh_db["Test Supplier User"]) + ) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index a1b13a246..59b32532a 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -17,7 +17,9 @@ def setup_api_test_data( Set up data for API v3.0 tests. """ print("Setting up data for API v3.0 tests on %s" % db.engine) - sensors = add_incineration_line(db, setup_roles_users["Test Supplier User"]) + sensors = add_incineration_line( + db, User.query.get(setup_roles_users["Test Supplier User"]) + ) return sensors @@ -32,7 +34,7 @@ def setup_api_fresh_test_data( for sensor in Sensor.query.all(): fresh_db.delete(sensor) sensors = add_incineration_line( - fresh_db, setup_roles_users_fresh_db["Test Supplier User"] + fresh_db, User.query.get(setup_roles_users_fresh_db["Test Supplier User"]) ) return sensors diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 5af1ec722..e570a33be 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -44,7 +44,9 @@ def test_get_sensor_data( ): """Check the /sensors/data endpoint for fetching 1 hour of data of a 10-minute resolution sensor.""" sensor = setup_api_test_data["some gas sensor"] - source: Source = setup_roles_users["Test Supplier User"].data_source[0] + source: Source = User.query.get( + setup_roles_users["Test Supplier User"] + ).data_source[0] assert sensor.event_resolution == timedelta(minutes=10) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", @@ -76,7 +78,9 @@ def test_get_instantaneous_sensor_data( ): """Check the /sensors/data endpoint for fetching 1 hour of data of an instantaneous sensor.""" sensor = setup_api_test_data["some temperature sensor"] - source: Source = setup_roles_users["Test Supplier User"].data_source[0] + source: Source = User.query.get( + setup_roles_users["Test Supplier User"] + ).data_source[0] assert sensor.event_resolution == timedelta(minutes=0) message = { "sensor": f"ea1.2021-01.io.flexmeasures:fm1.{sensor.id}", diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 5b1f8b90b..c40c2deba 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -98,7 +98,7 @@ def reporter_config_raw(app, db, setup_dummy_data): return reporter_config_raw -@pytest.mark.skip_github +# @pytest.mark.skip_github @pytest.fixture(scope="module") def process_power_sensor(db, app, add_market_prices): """ diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 77935deea..450ef013a 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -15,8 +15,10 @@ from flexmeasures.utils.time_utils import server_now -@pytest.mark.skip_github -def test_add_annotation(app, fresh_db, setup_roles_users_fresh_db): +# @pytest.mark.skip_github +# def test_add_annotation(app, fresh_db, setup_roles_users_fresh_db): +# @pytest.mark.skip_github +def test_add_annotation(app, db, setup_roles_users): from flexmeasures.cli.data_add import add_annotation cli_input = { @@ -51,8 +53,10 @@ def test_add_annotation(app, fresh_db, setup_roles_users_fresh_db): ) -@pytest.mark.skip_github -def test_add_holidays(app, fresh_db, setup_roles_users_fresh_db): +# @pytest.mark.skip_github +# def test_add_holidays(app, fresh_db, setup_roles_users_fresh_db): +# @pytest.mark.skip_github +def test_add_holidays(app, db, setup_roles_users): from flexmeasures.cli.data_add import add_holidays cli_input = { @@ -213,7 +217,7 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): assert len(stored_report) == 95 -@pytest.mark.skip_github +# @pytest.mark.skip_github @pytest.mark.parametrize("process_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) def test_add_process(app, process_power_sensor, process_type): """ diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 552a74420..ba23b61c8 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -227,7 +227,7 @@ def create_roles_users(db, test_accounts) -> dict[str, User]: ), ) ) - return {user.username: user for user in new_users} + return {user.username: user.id for user in new_users} @pytest.fixture(scope="module") @@ -401,12 +401,12 @@ def setup_assets( ) -> dict[str, Asset]: """Add assets to known test users. Deprecated. Remove with Asset model.""" - + # db.session.refresh(setup_roles_users["Test Prosumer User"]) assets = [] for asset_name in ["wind-asset-1", "wind-asset-2", "solar-asset-1"]: asset = Asset( name=asset_name, - owner_id=setup_roles_users["Test Prosumer User"].id, + owner_id=setup_roles_users["Test Prosumer User"], asset_type_name="wind" if "wind" in asset_name else "solar", event_resolution=timedelta(minutes=15), capacity_in_mw=1, @@ -440,6 +440,7 @@ def setup_assets( for dt, val in zip(time_slots, values) ] db.session.add_all(beliefs) + db.session.commit() return {asset.name: asset for asset in assets} @@ -591,7 +592,7 @@ def create_test_battery_assets( test_battery = Asset( name="Test battery", - owner_id=setup_roles_users["Test Prosumer User"].id, + owner_id=setup_roles_users["Test Prosumer User"], asset_type_name="battery", event_resolution=timedelta(minutes=15), capacity_in_mw=2, @@ -609,7 +610,7 @@ def create_test_battery_assets( test_battery_no_prices = Asset( name="Test battery with no known prices", - owner_id=setup_roles_users["Test Prosumer User"].id, + owner_id=setup_roles_users["Test Prosumer User"], asset_type_name="battery", event_resolution=timedelta(minutes=15), capacity_in_mw=2, @@ -677,7 +678,7 @@ def create_charging_station_assets( charging_station = Asset( name="Test charging station", - owner_id=setup_roles_users["Test Prosumer User"].id, + owner_id=setup_roles_users["Test Prosumer User"], asset_type_name="one-way_evse", event_resolution=timedelta(minutes=15), capacity_in_mw=2, @@ -695,7 +696,7 @@ def create_charging_station_assets( bidirectional_charging_station = Asset( name="Test charging station (bidirectional)", - owner_id=setup_roles_users["Test Prosumer User"].id, + owner_id=setup_roles_users["Test Prosumer User"], asset_type_name="two-way_evse", event_resolution=timedelta(minutes=15), capacity_in_mw=2, diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 98540df7b..41e065f29 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -8,6 +8,7 @@ from flask_sqlalchemy import SQLAlchemy from statsmodels.api import OLS +from flexmeasures import User from flexmeasures.data.models.annotations import Annotation from flexmeasures.data.models.assets import Asset from flexmeasures.data.models.data_sources import DataSource @@ -70,7 +71,7 @@ def setup_fresh_test_data( unit="MW", market_id=setup_markets["epex_da"].id, ) - asset.owner = setup_roles_users["Test Prosumer User"] + asset.owner = User.query.get(setup_roles_users["Test Prosumer User"]) db.session.add(asset) time_slots = pd.date_range( From dd1c186000c136f8cc0277fde261621e35da75e7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Aug 2023 14:06:25 +0200 Subject: [PATCH 45/48] add skip Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 2 +- flexmeasures/cli/tests/test_data_add.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index c40c2deba..5b1f8b90b 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -98,7 +98,7 @@ def reporter_config_raw(app, db, setup_dummy_data): return reporter_config_raw -# @pytest.mark.skip_github +@pytest.mark.skip_github @pytest.fixture(scope="module") def process_power_sensor(db, app, add_market_prices): """ diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 450ef013a..510b7d9ea 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -15,9 +15,7 @@ from flexmeasures.utils.time_utils import server_now -# @pytest.mark.skip_github -# def test_add_annotation(app, fresh_db, setup_roles_users_fresh_db): -# @pytest.mark.skip_github +@pytest.mark.skip_github def test_add_annotation(app, db, setup_roles_users): from flexmeasures.cli.data_add import add_annotation @@ -53,9 +51,7 @@ def test_add_annotation(app, db, setup_roles_users): ) -# @pytest.mark.skip_github -# def test_add_holidays(app, fresh_db, setup_roles_users_fresh_db): -# @pytest.mark.skip_github +@pytest.mark.skip_github def test_add_holidays(app, db, setup_roles_users): from flexmeasures.cli.data_add import add_holidays @@ -217,7 +213,7 @@ def test_add_reporter(app, db, setup_dummy_data, reporter_config_raw): assert len(stored_report) == 95 -# @pytest.mark.skip_github +@pytest.mark.skip_github @pytest.mark.parametrize("process_type", [("INFLEXIBLE"), ("SHIFTABLE"), ("BREAKABLE")]) def test_add_process(app, process_power_sensor, process_type): """ From 9c99196c3e86e62b7ee43d78e0e4986d4f2a32f2 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Aug 2023 18:03:37 +0200 Subject: [PATCH 46/48] fix fixture Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 82 +++++++++--------------------- 1 file changed, 23 insertions(+), 59 deletions(-) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 00ef96a1b..2fc5d4445 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -10,26 +10,42 @@ @pytest.fixture(scope="module") @pytest.mark.skip_github -def setup_dummy_data(db, app): +def setup_dummy_asset(db, app): """ - Create an asset with two sensors (1 and 2), and add the same set of 200 beliefs with an hourly resolution to each of them. - Return the two sensors and a result sensor (which has no data). + Create an Asset to add sensors to and return the id. """ - dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") - report_asset_type = GenericAssetType(name="ReportAssetType") - db.session.add_all([dummy_asset_type, report_asset_type]) + db.session.add(dummy_asset_type) dummy_asset = GenericAsset( name="DummyGenericAsset", generic_asset_type=dummy_asset_type ) + db.session.add(dummy_asset) + db.session.commit() + + return dummy_asset.id + + +@pytest.fixture(scope="module") +@pytest.mark.skip_github +def setup_dummy_data(db, app, setup_dummy_asset): + """ + Create an asset with two sensors (1 and 2), and add the same set of 200 beliefs with an hourly resolution to each of them. + Return the two sensors and a result sensor (which has no data). + """ + + report_asset_type = GenericAssetType(name="ReportAssetType") + + db.session.add(report_asset_type) pandas_report = GenericAsset( name="PandasReport", generic_asset_type=report_asset_type ) - db.session.add_all([dummy_asset, pandas_report]) + db.session.add(pandas_report) + + dummy_asset = GenericAsset.query.get(setup_dummy_asset) sensor1 = Sensor( "sensor 1", generic_asset=dummy_asset, event_resolution=timedelta(hours=1) @@ -96,55 +112,3 @@ def reporter_config_raw(app, db, setup_dummy_data): ) return reporter_config_raw - - -@pytest.mark.skip_github -@pytest.fixture(scope="module") -def process_power_sensor(db, app, add_market_prices): - """ - Create an asset of type "process", power sensor to hold the result of - the scheduler and price data consisting of 8 expensive hours, 8 cheap hours, and again 8 expensive hours- - - """ - - process_asset_type = GenericAssetType(name="process") - - db.session.add(process_asset_type) - - process_asset = GenericAsset( - name="Test Process Asset", generic_asset_type=process_asset_type - ) - - db.session.add(process_asset) - - power_sensor = Sensor( - "power", - generic_asset=process_asset, - event_resolution=timedelta(hours=1), - unit="MW", - ) - - db.session.add(power_sensor) - - db.session.commit() - - yield power_sensor.id - - -@pytest.fixture(scope="module") -@pytest.mark.skip_github -def setup_dummy_asset(db, app): - """ - Create an Asset to add sensors to and return the id. - """ - dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") - - db.session.add(dummy_asset_type) - - dummy_asset = GenericAsset( - name="DummyGenericAsset", generic_asset_type=dummy_asset_type - ) - db.session.add(dummy_asset) - db.session.commit() - - return dummy_asset.id From b02f01c9ece4723868568d31cc337f61f8deb8e6 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Aug 2023 18:13:03 +0200 Subject: [PATCH 47/48] add process fixture back Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/conftest.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/flexmeasures/cli/tests/conftest.py b/flexmeasures/cli/tests/conftest.py index 2fc5d4445..e83b53f07 100644 --- a/flexmeasures/cli/tests/conftest.py +++ b/flexmeasures/cli/tests/conftest.py @@ -112,3 +112,36 @@ def reporter_config_raw(app, db, setup_dummy_data): ) return reporter_config_raw + + +@pytest.mark.skip_github +@pytest.fixture(scope="module") +def process_power_sensor(db, app, add_market_prices): + """ + Create an asset of type "process", power sensor to hold the result of + the scheduler and price data consisting of 8 expensive hours, 8 cheap hours, and again 8 expensive hours- + + """ + + process_asset_type = GenericAssetType(name="process") + + db.session.add(process_asset_type) + + process_asset = GenericAsset( + name="Test Process Asset", generic_asset_type=process_asset_type + ) + + db.session.add(process_asset) + + power_sensor = Sensor( + "power", + generic_asset=process_asset, + event_resolution=timedelta(hours=1), + unit="MW", + ) + + db.session.add(power_sensor) + + db.session.commit() + + yield power_sensor.id From d0b7eb7df91d4bbdde2727f748165d3aca6839ae Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 1 Aug 2023 18:22:56 +0200 Subject: [PATCH 48/48] add skip Signed-off-by: Victor Garcia Reolid --- flexmeasures/cli/tests/test_data_add.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/cli/tests/test_data_add.py b/flexmeasures/cli/tests/test_data_add.py index 17a08ab0c..995cc706f 100644 --- a/flexmeasures/cli/tests/test_data_add.py +++ b/flexmeasures/cli/tests/test_data_add.py @@ -256,6 +256,7 @@ def test_add_process(app, process_power_sensor, process_type): assert (schedule == -0.4).event_value.sum() == 4 +@pytest.mark.skip_github @pytest.mark.parametrize( "event_resolution, name, success", [("PT20M", "ONE", True), (15, "TWO", True), ("some_string", "THREE", False)],